class_docstr: Base class for yet to train and pretrained tokenizers
  
  Implements the ``vocabulary_`` attribute and the methods
  ``get_feature_names``, ``transform``, and ``inverse_transform``.
  
  Subclasses should implement the ``fit``  method.
class_docstr_tok: ["Base","class","for","yet","to","train","and","pretrained","tokenizers","Implements","the","``","vocabulary_","``","attribute","and","the","methods","``","get_feature_names","``",",","``","transform","``",",","and","``","inverse_transform","``",".","Subclasses","should","implement","the","``","fit","``","method","."]
docstr: Transform the given data
  
  Parameters
  ----------
  X : iterable of str
    A list/array of strings or an iterable which generates either strings.
  
  Returns
  -------
  Xt : transformers.tokenization_utils_base.BatchEncoding
    A Hugging Face ``BatchEncoding`` instance. This is basically a
    dictionary containing the ids of the tokens and some additional fields
    (depending on the parameters), e.g. the attention mask. For this
    reason, the output is not well suited to be used with normal sklearn
    models, but it works with Hugging Face transformers and with skorch
    nets.
docstr_tok: ["Transform","the","given","data","Parameters","--","--","--","--","--","X",":","iterable","of","str","A","list\/array","of","strings","or","an","iterable","which","generates","either","strings",".","Returns","--","--","--","-","Xt",":","transformers.tokenization_utils_base.BatchEncoding","A","Hugging","Face","``","BatchEncoding","``","instance",".","This","is","basically","a","dictionary","containing","the","ids","of","the","tokens","and","some","additional","fields","(","depending","on","the","parameters",")",",","e.g",".","the","attention","mask",".","For","this","reason",",","the","output","is","not","well","suited","to","be","used","with","normal","sklearn","models",",","but","it","works","with","Hugging","Face","transformers","and","with","skorch","nets","."]
code: def transform(self, X):
      """Transform the given data
  
      Parameters
      ----------
      X : iterable of str
        A list/array of strings or an iterable which generates either strings.
  
      Returns
      -------
      Xt : transformers.tokenization_utils_base.BatchEncoding
        A Hugging Face ``BatchEncoding`` instance. This is basically a
        dictionary containing the ids of the tokens and some additional fields
        (depending on the parameters), e.g. the attention mask. For this
        reason, the output is not well suited to be used with normal sklearn
        models, but it works with Hugging Face transformers and with skorch
        nets.
  
      """
      check_is_fitted(self)
  
      # from sklearn, triggers a parameter validation
      if isinstance(X, str):
          raise ValueError(
              "Iterable over raw text documents expected, string object received."
          )
      X = list(X)  # transformers tokenizer does not accept arrays
  
      verbose = bool(self.verbose)
  
      # When using tensors/arrays, truncate/pad to max length, otherwise don't
      return_tensors = None
      truncation = False
      padding = False
      if self.return_tensors is not None:
          return_tensors = self.return_tensors
          padding = "max_length"
          truncation = True
  
      Xt = self.fast_tokenizer_(
          X,
          max_length=self.max_length,
          padding=padding,
          truncation=truncation,
          return_tensors=return_tensors,
          return_token_type_ids=self.return_token_type_ids,
          return_length=self.return_length,
          return_attention_mask=self.return_attention_mask,
          verbose=verbose,
      )
      return Xt
  
imports: import io
  import os
  import pathlib
  from copy import deepcopy
  from operator import itemgetter
  import numpy
  import torch
  from sklearn.base import BaseEstimator, TransformerMixin
  from skorch.callbacks import LRScheduler
  from skorch.dataset import unpack_data
  from skorch.utils import check_is_fitted, params_for
label_desc: Use image node_id 5 for calling the _HuggingfaceTokenizerBase obj's underlying member method code with example usage: obj.transform(X) and returns: Xt