docstr: Split the text in head and tail. Some language models support a maximum
  number of tokens. Thus is necessary to split the text to meet this constraint.
  After the text is split in head and tail, only the head is considered for operation.
  Thus the tail will remain unchanged.
  
  Parameters
  ----------
  text
      Text to be split in head and tail.
  
  Returns
  -------
  Tuple consisting of the head, tail and their corresponding list of tokens.
docstr_tok: ["Split","the","text","in","head","and","tail",".","Some","language","models","support","a","maximum","number","of","tokens",".","Thus","is","necessary","to","split","the","text","to","meet","this","constraint",".","After","the","text","is","split","in","head","and","tail",",","only","the","head","is","considered","for","operation",".","Thus","the","tail","will","remain","unchanged",".","Parameters","--","--","--","--","--","text","Text","to","be","split","in","head","and","tail",".","Returns","--","--","--","-","Tuple","consisting","of","the","head",",","tail","and","their","corresponding","list","of","tokens","."]
code: def head_tail_split(
      self, text: str
  ) -> Tuple[str, str, List[str], List[str]]:
      """
      Split the text in head and tail. Some language models support a maximum
      number of tokens. Thus is necessary to split the text to meet this constraint.
      After the text is split in head and tail, only the head is considered for operation.
      Thus the tail will remain unchanged.
  
      Parameters
      ----------
      text
          Text to be split in head and tail.
  
      Returns
      -------
      Tuple consisting of the head, tail and their corresponding list of tokens.
      """
      text = text.strip()
      if len(text) == 0:
          raise ValueError("The text is empty.")
  
      # data = `This is not a wordy sentence` -> tokens = [this, is, not, a, word, ##y, sentence, .]
      tokens: List[str] = self.tokenizer.tokenize(text)
  
      # some models do not have a max length restrictions (e.g. XLNet)
      if (
          self.max_num_tokens == -1
          or len(tokens) <= self.max_num_tokens
      ):
          return text, "", tokens, []
  
      # head's length
      head_num_tokens = self.max_num_tokens
  
      # decrease the head length so it contains full words
      while (head_num_tokens > 0) and self.is_subword_prefix(
          tokens[head_num_tokens]
      ):
          head_num_tokens -= 1
  
      if head_num_tokens == 0:
          raise ValueError(
              "Check the first word in the sentence. Seems it is a very long word"
          )
  
      ids = self.tokenizer.convert_tokens_to_ids(
          tokens[:head_num_tokens]
      )
      head_text = self.tokenizer.decode(ids).strip()
      tail_text = ""
  
      # if the number of tokens exceeds the maximum allowed
      # number, then construct also the tail_text
      if len(tokens) >= head_num_tokens:
          ids = self.tokenizer.convert_tokens_to_ids(
              tokens[head_num_tokens:]
          )
          tail_text = self.tokenizer.decode(ids).strip()
  
      return (
          head_text,
          tail_text,
          tokens[:head_num_tokens],
          tokens[head_num_tokens:],
      )
  
imports: import abc
  import numpy
  from pathlib import Path
  from typing import Any, Callable, List, Optional, Tuple, Union
  import tensorflow
  import transformers
  from transformers import TFAutoModelForMaskedLM, AutoTokenizer
label_desc: Use image node_id 11 for calling the LanguageModel obj's underlying member method code with example usage: obj.head_tail_split(text) and returns: head_text,  tail_text,  tokens,  tokens, text,  str,  tokens,  list