code: def _get_features(
      queries: Union[List[str], Tuple[str, ...]],
      punct_label_lines: Union[List[str], Tuple[str, ...]],
      capit_label_lines: Union[List[str], Tuple[str, ...]],
      max_seq_length: int,
      tokenizer: TokenizerSpec,
      punct_label_ids: Dict[str, int] = None,
      capit_label_ids: Dict[str, int] = None,
      pad_label: str = "O",
      verbose: bool = True,
      n_jobs: Optional[int] = 0,
      progress_queue: Optional[mp.Queue] = None,
      audio_queries: Optional[List[str]] = None,
      sample_rate: Optional[int] = None,
      preload_audios: Optional[bool] = True,
  ) -> Tuple[
      List[Any],
      List[Any],
      List[Any],
      List[Any],
      List[Any],
      List[Any],
      List[Any],
  ]:
      """
      Tokenizes data, encodes labels, creates masks of first tokens in words, clips sequences by number of tokens.
  
      Args:
          queries: text sequences
          max_seq_length: max number of tokens in an input sequence including [CLS] and [SEP] tokens. If number of tokens
              in a sequence exceeds ``max_seq_length``, then excess tokens in the end of the sequence are removed
          tokenizer: a tokenizer instance which has properties ``cls_id``, ``pad_id``, ``sep_id``, ``unk_id``
          punct_label_ids: dict to map punctuation labels to label ids. Starts with pad_label->0.
          capit_label_ids: dict to map capitalization labels to label ids. Starts with pad_label->0.
          pad_label: pad value use for labels. By default, it's the neutral label for punctuation and capitalization.
              Its id in ``punct_label_ids`` and ``capit_label_ids`` has to be ``0``
          punct_label_lines: a list of a tuple of labels for every word in a sequence (str)
          capit_label_lines: a list or a tuple of labels for every word in a sequence (str)
          verbose: whether to show examples of tokenized data and various progress information
          n_jobs: a number of workers used for preparing features. If ``n_jobs <= 0``, then do not use multiprocessing
              and run features creation in this process. If not set, number of workers will be equal to the number of
              CPUs.
  
              !!WARNING!!
              There can be deadlocking problems with some tokenizers (e.g. SentencePiece, HuggingFace AlBERT)
              if ``n_jobs > 0``.
  
          progress_queue: a multiprocessing queue used for reporting progress. Useful for creating tarred dataset
          audio_queries: a list of audio filepaths
          sample_rate: target sample rate of audios
          preload_audios: whether to preload audios or not
  
      Returns:
          input_ids: a list of 1D int32 arrays. Each array contains token ids of corresponding query
          subtokens_mask: a list of 1D boolean arrays. An array element is ``True`` if corresponding token is the
              first token in a word
          punct_labels: a list of 1D int32 arrays. Encoded punctuation labels for every token in a query. Tokens in one
              word have identical labels.
          capit_labels: a list of 1D int32 arrays. Encoded capitalization labels for every token in a query. Tokens in
              one word have identical labels
      """
      if verbose:
          logging.info("Start initial tokenization.")
      create_progress_process = progress_queue is None
      if n_jobs is None:
          n_jobs = min(mp.cpu_count(), len(queries))
  
      if verbose:
          logging.info(f"Running tokenization with {n_jobs} jobs.")
  
      # Number of queries in split
      split_size = min(
          len(queries) // max(n_jobs, 1), MAX_NUM_QUERIES_IN_SPLIT
      )
      n_split = len(queries) // split_size
      split_queries = [
          queries[split_size * i : split_size * (i + 1)]
          for i in range(n_split - 1)
      ] + [queries[split_size * (n_split - 1) :]]
      split_punct_labels_lines = [
          punct_label_lines[split_size * i : split_size * (i + 1)]
          for i in range(n_split - 1)
      ] + [punct_label_lines[split_size * (n_split - 1) :]]
      split_capit_labels_lines = [
          capit_label_lines[split_size * i : split_size * (i + 1)]
          for i in range(n_split - 1)
      ] + [capit_label_lines[split_size * (n_split - 1) :]]
  
      args = list(
          zip(
              split_queries,
              split_punct_labels_lines,
              split_capit_labels_lines,
              range(n_split),
          )
      )
      if audio_queries:
          split_audio_queries = [
              audio_queries[split_size * i : split_size * (i + 1)]
              for i in range(n_split - 1)
          ] + [audio_queries[split_size * (n_split - 1) :]]
  
          args = list(
              zip(
                  split_queries,
                  split_punct_labels_lines,
                  split_capit_labels_lines,
                  range(n_split),
                  split_audio_queries,
                  [sample_rate for _ in range(n_split)],
                  [preload_audios for _ in range(n_split)],
              )
          )
      if create_progress_process:
          progress = Progress(len(queries), "Tokenization", "query")
          progress_queue = progress.get_queues()[0]
      if n_jobs > 0:
          with mp.Pool(n_jobs) as pool:
              result = pool.starmap(
                  TokenizeCreateMasksClipWorker(
                      max_seq_length,
                      tokenizer,
                      punct_label_ids,
                      capit_label_ids,
                      pad_label,
                      verbose,
                      progress_queue,
                  ),
                  args,
              )
      else:
          result = []
          for x in args:
              result.append(
                  TokenizeCreateMasksClipWorker(
                      max_seq_length,
                      tokenizer,
                      punct_label_ids,
                      capit_label_ids,
                      pad_label,
                      verbose,
                      progress_queue,
                  )(*x)
              )
      if create_progress_process:
          progress.finish()
  
      (
          input_ids,
          subtokens_mask,
          punct_labels,
          capit_labels,
          waveforms,
          audio_lengths,
          audio_filepaths,
      ) = tuple(list(itertools.chain(*e)) for e in zip(*result))
      if verbose:
          logging.info("Finished initial tokenization.")
          get_stats([len(inp) for inp in input_ids])
          logging.info(f"Finished clipping and padding.")
          for i in range(min(len(input_ids), 5)):
              logging.info("*** Example ***")
              logging.info("i: %s" % i)
              logging.info(
                  "subtokens: %s"
                  % " ".join(list(map(str, input_ids[i])))
              )
              logging.info(
                  "subtokens_mask: %s"
                  % " ".join(list(map(str, subtokens_mask[i])))
              )
              logging.info(
                  "punct_labels: %s"
                  % " ".join(list(map(str, punct_labels[i])))
              )
              logging.info(
                  "capit_labels: %s"
                  % " ".join(list(map(str, capit_labels[i])))
              )
  
      return (
          input_ids,
          subtokens_mask,
          waveforms,
          audio_lengths,
          audio_filepaths,
          punct_labels,
          capit_labels,
      )
  
imports: import itertools
  import multiprocessing
  import os
  import pickle
  import tempfile
  from dataclasses import dataclass
  from math import ceil
  from pathlib import Path
  from queue import Empty
  from time import sleep
  from typing import Any, Dict, List, Optional, Set, Tuple, Union
  import numpy
  import torch
  from numpy import ndarray
  from omegaconf import MISSING, DictConfig, OmegaConf
  from torch.nn.utils.rnn import pad_sequence
  from tqdm import tqdm
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils.data_preprocessing import get_label_stats, get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import AudioSignal, ChannelType, LabelsType, LengthsType, MaskType, NeuralType
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 5 for calling a global function with example usage: _get_features(queries, punct_label_lines, capit_label_lines, max_seq_length, tokenizer, punct_label_ids, capit_label_ids, pad_label, verbose, n_jobs, progress_queue, audio_queries, sample_rate, preload_audios) and returns: input_ids,  subtokens_mask,  waveforms,  audio_lengths,  audio_filepaths,  punct_labels,  capit_labels