class_docstr: Base class for text encoders.
  Translates raw texts into structured token arrays
class_docstr_tok: ["Base","class","for","text","encoders",".","Translates","raw","texts","into","structured","token","arrays"]
docstr: Takes some tokenized text and arranges it into a format that maximises the amount of kept text from each
  whilst keeping the overall sequence length within max_length tokens. It also adds the 3 special tokens. Start,
   Classify and Delimiter.
  :param encoded: Lists of shape [batch, n_fields, num_tokens]
  :param max_length: Int representing the max length of a single sample
  :param start: Override the default start token.
  :param delimiter: Override the default delimiter token.
  :param end: Override the default classify token
  :return: Formatted outputs of the form. [batch, num_tokens] where num_tokens' <= max_length
docstr_tok: ["Takes","some","tokenized","text","and","arranges","it","into","a","format","that","maximises","the","amount","of","kept","text","from","each","whilst","keeping","the","overall","sequence","length","within","max_length","tokens",".","It","also","adds","the","3","special","tokens",".","Start",",","Classify","and","Delimiter",".",":","param","encoded",":","Lists","of","shape","[","batch",",","n_fields",",","num_tokens","]",":","param","max_length",":","Int","representing","the","max","length","of","a","single","sample",":","param","start",":","Override","the","default","start","token",".",":","param","delimiter",":","Override","the","default","delimiter","token",".",":","param","end",":","Override","the","default","classify","token",":","return",":","Formatted","outputs","of","the","form",".","[","batch",",","num_tokens","]","where","num_tokens","'","<","=","max_length"]
code: def _cut_and_concat(
      self,
      *,
      encoded,
      max_length,
      special_tokens=None,
      start=None,
      delimiter=None,
      end=None,
      include_bos_eos=True,
      eos_on_cut=True,
  ):
      """
      Takes some tokenized text and arranges it into a format that maximises the amount of kept text from each
      whilst keeping the overall sequence length within max_length tokens. It also adds the 3 special tokens. Start,
       Classify and Delimiter.
      :param encoded: Lists of shape [batch, n_fields, num_tokens]
      :param max_length: Int representing the max length of a single sample
      :param start: Override the default start token.
      :param delimiter: Override the default delimiter token.
      :param end: Override the default classify token
      :return: Formatted outputs of the form. [batch, num_tokens] where num_tokens' <= max_length
      """
      start = start or special_tokens or self.start_token
      delimiter = delimiter or special_tokens or self.delimiter_token
      clf_token = end or special_tokens or self.end_token
      num_samples = len(encoded)
      adjusted_max_length = max_length - num_samples - 1
      allocated_max_len = adjusted_max_length // num_samples
  
      overflows = [
          allocated_max_len - len(sequence) for sequence in encoded
      ]
      spare = sum(overflows)
  
      if spare >= 0:
          cut_len = None
      else:
          warnings.warn(
              "Document is longer than max length of the model, trimming document from {} to {} tokens. Try chunk_long_sequences=True".format(
                  -spare + max_length, max_length
              )
          )
          empty_tokens = sum(max(overflow, 0) for overflow in overflows)
          num_over = [max(overflow, 0) for overflow in overflows].count(
              0
          )
          if num_over == 0:
              cut_len = allocated_max_len
          else:
              cut_len = allocated_max_len + (empty_tokens // num_over)
  
      if include_bos_eos == True or include_bos_eos == "bos":
          joined = [start]
      else:
          joined = []
  
      for d in encoded:
          joined += d[:cut_len] + [delimiter]
      joined = joined[:-1]
  
      if include_bos_eos == True or include_bos_eos == "eos":
          if eos_on_cut or cut_len is None:
              joined += [clf_token]
      return joined
  
imports: import warnings
  import functools
  from collections import namedtuple, Counter, OrderedDict
  import spacy
  import numpy
label_desc: Use image node_id 7 for calling the BaseEncoder obj's underlying member method code with example usage: obj._cut_and_concat() and returns: joined