class_docstr: A worker for tokenization, encoding labels, creating masks for first token in a word, sequence clipping
class_docstr_tok: ["A","worker","for","tokenization",",","encoding","labels",",","creating","masks","for","first","token","in","a","word",",","sequence","clipping"]
docstr: Tokenize, clip, encode labels, and create masks of first tokens in words.
  
  Args:
      queries: text sequences
      punct_label_lines: a list or a tuple of labels for every word in a sequence (str)
      capit_label_lines: a list of a tuple labels for every word in a sequence (str)
      split_i: number of a split which is processed. Used for logging
      audio_queries: a list of audio filepaths
      sample_rate: target sample rate of audios
      preload_audios: whether to preload audios or not
  
  Returns:
      input_ids: a list of 1D int32 arrays. Each array contains token ids of the corresponding query
      subtokens_mask: a list of 1D boolean arrays. An array element is ``True`` if corresponding token is the
          first token in a word
      punct_labels: a list of 1D int32 arrays. Encoded punctuation labels for every token in a query. Tokens in
          one word have identical labels
      capit_labels: a list of 1D int32 arrays. Encoded capitalization labels for every token in a query. Tokens
          in one word have identical labels
docstr_tok: ["Tokenize",",","clip",",","encode","labels",",","and","create","masks","of","first","tokens","in","words",".","Args",":","queries",":","text","sequences","punct_label_lines",":","a","list","or","a","tuple","of","labels","for","every","word","in","a","sequence","(","str",")","capit_label_lines",":","a","list","of","a","tuple","labels","for","every","word","in","a","sequence","(","str",")","split_i",":","number","of","a","split","which","is","processed",".","Used","for","logging","audio_queries",":","a","list","of","audio","filepaths","sample_rate",":","target","sample","rate","of","audios","preload_audios",":","whether","to","preload","audios","or","not","Returns",":","input_ids",":","a","list","of","1D","int32","arrays",".","Each","array","contains","token","ids","of","the","corresponding","query","subtokens_mask",":","a","list","of","1D","boolean","arrays",".","An","array","element","is","``","True","``","if","corresponding","token","is","the","first","token","in","a","word","punct_labels",":","a","list","of","1D","int32","arrays",".","Encoded","punctuation","labels","for","every","token","in","a","query",".","Tokens","in","one","word","have","identical","labels","capit_labels",":","a","list","of","1D","int32","arrays",".","Encoded","capitalization","labels","for","every","token","in","a","query",".","Tokens","in","one","word","have","identical","labels"]
code: def __call__(
      self,
      queries: List[str],
      punct_label_lines: Optional[Union[List[str], Tuple[str, ...]]],
      capit_label_lines: Optional[Union[List[str], Tuple[str, ...]]],
      split_i: int,
      audio_queries: Optional[List[str]] = None,
      sample_rate: Optional[int] = None,
      preload_audios: Optional[bool] = True,
  ) -> Tuple[
      List[ndarray],
      List[ndarray],
      List[ndarray],
      List[ndarray],
      Union[List[Any], List[None]],
      Union[List[Any], List[None]],
      Union[List[Any], List[None]],
  ]:
      """
      Tokenize, clip, encode labels, and create masks of first tokens in words.
  
      Args:
          queries: text sequences
          punct_label_lines: a list or a tuple of labels for every word in a sequence (str)
          capit_label_lines: a list of a tuple labels for every word in a sequence (str)
          split_i: number of a split which is processed. Used for logging
          audio_queries: a list of audio filepaths
          sample_rate: target sample rate of audios
          preload_audios: whether to preload audios or not
  
      Returns:
          input_ids: a list of 1D int32 arrays. Each array contains token ids of the corresponding query
          subtokens_mask: a list of 1D boolean arrays. An array element is ``True`` if corresponding token is the
              first token in a word
          punct_labels: a list of 1D int32 arrays. Encoded punctuation labels for every token in a query. Tokens in
              one word have identical labels
          capit_labels: a list of 1D int32 arrays. Encoded capitalization labels for every token in a query. Tokens
              in one word have identical labels
      """
      (
          all_input_ids,
          all_subtokens_mask,
          punct_all_labels,
          capit_all_labels,
      ) = ([], [], [], [])
      dummy = [None] * len(
          queries
      )  # Needed to avoid code duplication with different values of `self.use_audio`
      all_audio_waveforms = [] if preload_audios else dummy
      audio_lengths = [] if preload_audios else dummy
      audio_filepaths = [] if not preload_audios else dummy
      progress_made = 0
      queries = (
          zip(queries, audio_queries)
          if audio_queries
          else zip(queries, dummy)
      )
      for i, (query, audio_query) in enumerate(queries):
          words = query.split()
          input_ids, subtokens_mask = [self.tokenizer.cls_id], [0]
          _check_number_of_labels(
              words,
              query,
              i,
              split_i,
              punct_label_lines[i],
              capit_label_lines[i],
          )
          pad_id = self.punct_label_ids[self.pad_label]
          punct_labels = [pad_id]
          punct_query_labels = [
              self.punct_label_ids[lab] for lab in punct_label_lines[i]
          ]
          capit_labels = [pad_id]
          capit_query_labels = [
              self.capit_label_ids[lab] for lab in capit_label_lines[i]
          ]
          for j, word in enumerate(words):
              word_ids = self.tokenizer.text_to_ids(word)
              if not word_ids and len(word):
                  word_ids = [self.tokenizer.unk_id]
              input_ids.extend(word_ids)
  
              subtokens_mask.append(1)
              subtokens_mask.extend([0] * (len(word_ids) - 1))
  
              punct_labels.extend(
                  [punct_query_labels[j]] * len(word_ids)
              )
              capit_labels.extend(
                  [capit_query_labels[j]] * len(word_ids)
              )
  
          # add eos token
          input_ids.append(self.tokenizer.sep_id)
          subtokens_mask.append(0)
  
          all_input_ids.append(
              np.array(
                  self._maybe_clip(input_ids, self.tokenizer.sep_id),
                  dtype=np.int32,
              )
          )
          all_subtokens_mask.append(
              np.array(self._maybe_clip(subtokens_mask, 0), dtype=bool)
          )
  
          punct_labels.append(pad_id)
          punct_all_labels.append(
              np.array(
                  self._maybe_clip(punct_labels, pad_id), dtype=np.int32
              )
          )
          capit_labels.append(pad_id)
          capit_all_labels.append(
              np.array(
                  self._maybe_clip(capit_labels, pad_id), dtype=np.int32
              )
          )
          if preload_audios and audio_query:
              if ASR_AVAILABLE:
                  segment = AudioSegment.from_file(
                      audio_query.strip(), target_sr=sample_rate
                  )
                  all_audio_waveforms.append(segment.samples)
                  audio_lengths.append(segment.num_samples)
              else:
                  raise ModuleNotFoundError(
                      "Nemo ASR was not installed, see https://github.com/NVIDIA/NeMo#installation for installation instructions"
                  )
  
          elif audio_query:
              audio_filepaths.append(audio_query.strip())
  
          progress_made += 1
          if progress_made >= TOKENIZATION_PROGRESS_REPORT_PERIOD:
              self.progress_queue.put(progress_made)
              progress_made = 0
  
      self.progress_queue.put(progress_made)
      if self.verbose:
          logging.info(
              f"Finished processing data split number {split_i}"
          )
  
      return (
          all_input_ids,
          all_subtokens_mask,
          punct_all_labels,
          capit_all_labels,
          all_audio_waveforms,
          audio_lengths,
          audio_filepaths,
      )
  
imports: import itertools
  import multiprocessing
  import os
  import pickle
  import tempfile
  from dataclasses import dataclass
  from math import ceil
  from pathlib import Path
  from queue import Empty
  from time import sleep
  from typing import Any, Dict, List, Optional, Set, Tuple, Union
  import numpy
  import torch
  from numpy import ndarray
  from omegaconf import MISSING, DictConfig, OmegaConf
  from torch.nn.utils.rnn import pad_sequence
  from tqdm import tqdm
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils.data_preprocessing import get_label_stats, get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import AudioSignal, ChannelType, LabelsType, LengthsType, MaskType, NeuralType
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 3 for calling the TokenizeCreateMasksClipWorker obj's underlying member method code with example usage: obj.__call__(queries, punct_label_lines, capit_label_lines, split_i, audio_queries, sample_rate, preload_audios) and returns: all_input_ids,  all_subtokens_mask,  punct_all_labels,  capit_all_labels,  all_audio_waveforms,  audio_lengths,  audio_filepaths