class_docstr: A worker for tokenization, encoding labels, creating masks for first token in a word, sequence clipping
class_docstr_tok: ["A","worker","for","tokenization",",","encoding","labels",",","creating","masks","for","first","token","in","a","word",",","sequence","clipping"]
docstr: Args:
      max_seq_length: max number of tokens in an input sequence including [CLS] and [SEP] tokens. If number of
          tokens in a sequence exceeds ``max_seq_length``, then excess tokens in the end of the sequence
          are removed
      tokenizer: a tokenizer instance which has properties ``cls_id``, ``pad_id``, ``sep_id``, ``unk_id``
      punct_label_ids: dict to map punctuation labels to label ids. Starts with pad_label->0.
      capit_label_ids: dict to map capitalization labels to label ids. Starts with pad_label->0.
      pad_label: pad value use for labels. By default, it's the neutral label for punctuation and capitalization.
          Its id in ``punct_label_ids`` and ``capit_label_ids`` has to be ``0``
      verbose: whether to report when the worker finishes its job
      progress_queue: a multiprocessing queue used for reporting progress. Useful for creating tarred dataset
docstr_tok: ["Args",":","max_seq_length",":","max","number","of","tokens","in","an","input","sequence","including","[","CLS","]","and","[","SEP","]","tokens",".","If","number","of","tokens","in","a","sequence","exceeds","``","max_seq_length","``",",","then","excess","tokens","in","the","end","of","the","sequence","are","removed","tokenizer",":","a","tokenizer","instance","which","has","properties","``","cls_id","``",",","``","pad_id","``",",","``","sep_id","``",",","``","unk_id","``","punct_label_ids",":","dict","to","map","punctuation","labels","to","label","ids",".","Starts","with","pad_label-",">","0",".","capit_label_ids",":","dict","to","map","capitalization","labels","to","label","ids",".","Starts","with","pad_label-",">","0",".","pad_label",":","pad","value","use","for","labels",".","By","default",",","it","'s","the","neutral","label","for","punctuation","and","capitalization",".","Its","id","in","``","punct_label_ids","``","and","``","capit_label_ids","``","has","to","be","``","0","``","verbose",":","whether","to","report","when","the","worker","finishes","its","job","progress_queue",":","a","multiprocessing","queue","used","for","reporting","progress",".","Useful","for","creating","tarred","dataset"]
code: def __init__(
      self,
      max_seq_length: int,
      tokenizer: TokenizerSpec,
      punct_label_ids: Optional[Dict[str, int]],
      capit_label_ids: Optional[Dict[str, int]],
      pad_label: str,
      verbose: bool,
      progress_queue: mp.Queue,
  ) -> None:
      """
      Args:
          max_seq_length: max number of tokens in an input sequence including [CLS] and [SEP] tokens. If number of
              tokens in a sequence exceeds ``max_seq_length``, then excess tokens in the end of the sequence
              are removed
          tokenizer: a tokenizer instance which has properties ``cls_id``, ``pad_id``, ``sep_id``, ``unk_id``
          punct_label_ids: dict to map punctuation labels to label ids. Starts with pad_label->0.
          capit_label_ids: dict to map capitalization labels to label ids. Starts with pad_label->0.
          pad_label: pad value use for labels. By default, it's the neutral label for punctuation and capitalization.
              Its id in ``punct_label_ids`` and ``capit_label_ids`` has to be ``0``
          verbose: whether to report when the worker finishes its job
          progress_queue: a multiprocessing queue used for reporting progress. Useful for creating tarred dataset
      """
      self.max_seq_length = max_seq_length
      self.tokenizer = tokenizer
      self.punct_label_ids = punct_label_ids
      self.capit_label_ids = capit_label_ids
      self.pad_label = pad_label
      self.verbose = verbose
      self.progress_queue = progress_queue
  
imports: import itertools
  import multiprocessing
  import os
  import pickle
  import tempfile
  from dataclasses import dataclass
  from math import ceil
  from pathlib import Path
  from queue import Empty
  from time import sleep
  from typing import Any, Dict, List, Optional, Set, Tuple, Union
  import numpy
  import torch
  from numpy import ndarray
  from omegaconf import MISSING, DictConfig, OmegaConf
  from torch.nn.utils.rnn import pad_sequence
  from tqdm import tqdm
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils.data_preprocessing import get_label_stats, get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import AudioSignal, ChannelType, LabelsType, LengthsType, MaskType, NeuralType
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 1 to create a new TokenizeCreateMasksClipWorker object  with example: obj = TokenizeCreateMasksClipWorker(max_seq_length, tokenizer, punct_label_ids, capit_label_ids, pad_label, verbose, progress_queue)