class_docstr: Transformer word tagging classifier
  Args:
      model_type(str): model family (classifier head), choose between bert/quant_bert/xlnet
      labels (List[str], optional): list of tag labels
class_docstr_tok: ["Transformer","word","tagging","classifier","Args",":","model_type","(","str",")",":","model","family","(","classifier","head",")",",","choose","between","bert\/quant_bert\/xlnet","labels","(","List","[","str","]",",","optional",")",":","list","of","tag","labels"]
docstr: Loads a data file into a list of `InputBatch`s
  `cls_token_at_end` define the location of the CLS token:
      - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
      - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
  `cls_token_segment_id` define the segment id associated to the CLS token
  (0 for BERT, 2 for XLNet)
docstr_tok: ["Loads","a","data","file","into","a","list","of","`","InputBatch","`","s","`","cls_token_at_end","`","define","the","location","of","the","CLS","token",":","-","False","(","Default",",","BERT\/XLM","pattern",")",":","[","CLS","]","+","A","+","[","SEP","]","+","B","+","[","SEP","]","-","True","(","XLNet\/GPT","pattern",")",":","A","+","[","SEP","]","+","B","+","[","SEP","]","+","[","CLS","]","`","cls_token_segment_id","`","define","the","segment","id","associated","to","the","CLS","token","(","0","for","BERT",",","2","for","XLNet",")"]
code: def _convert_examples_to_features(
      self,
      examples: List[TokenClsInputExample],
      max_seq_length,
      tokenizer,
      include_labels=True,
      cls_token_at_end=False,
      pad_on_left=False,
      cls_token="[CLS]",
      sep_token="[SEP]",
      pad_token=0,
      sequence_segment_id=0,
      sep_token_extra=0,
      cls_token_segment_id=1,
      pad_token_segment_id=0,
      mask_padding_with_zero=True,
  ):
      """Loads a data file into a list of `InputBatch`s
      `cls_token_at_end` define the location of the CLS token:
          - False (Default, BERT/XLM pattern): [CLS] + A + [SEP] + B + [SEP]
          - True (XLNet/GPT pattern): A + [SEP] + B + [SEP] + [CLS]
      `cls_token_segment_id` define the segment id associated to the CLS token
      (0 for BERT, 2 for XLNet)
      """
  
      if include_labels:
          label_map = {v: k for k, v in self.labels_id_map.items()}
          label_pad = 0
  
      features = []
      for ex_index, example in enumerate(examples):
          if ex_index % 10000 == 0:
              logger.info(
                  "Processing example %d of %d", ex_index, len(examples)
              )
  
          tokens = []
          labels = []
          valid_tokens = []
          for i, token in enumerate(example.tokens):
              new_tokens = tokenizer.tokenize(token)
              tokens.extend(new_tokens)
              v_tok = [0] * (len(new_tokens))
              v_tok[0] = 1
              valid_tokens.extend(v_tok)
              if include_labels:
                  v_lbl = [label_pad] * (len(new_tokens))
                  v_lbl[0] = label_map.get(example.label[i])
                  labels.extend(v_lbl)
  
          # truncate by max_seq_length
          special_tokens_count = 3 if sep_token_extra else 2
          tokens = tokens[: (max_seq_length - special_tokens_count)]
          valid_tokens = valid_tokens[
              : (max_seq_length - special_tokens_count)
          ]
          if include_labels:
              labels = labels[: (max_seq_length - special_tokens_count)]
  
          tokens += [sep_token]
          if include_labels:
              labels += [label_pad]
          valid_tokens += [0]
          if sep_token_extra:  # roberta special case
              tokens += [sep_token]
              valid_tokens += [0]
              if include_labels:
                  labels += [label_pad]
          segment_ids = [sequence_segment_id] * len(tokens)
  
          if cls_token_at_end:
              tokens = tokens + [cls_token]
              segment_ids = segment_ids + [cls_token_segment_id]
              if include_labels:
                  labels = labels + [label_pad]
              valid_tokens = valid_tokens + [0]
          else:
              tokens = [cls_token] + tokens
              segment_ids = [cls_token_segment_id] + segment_ids
              if include_labels:
                  labels = [label_pad] + labels
              valid_tokens = [0] + valid_tokens
  
          input_ids = tokenizer.convert_tokens_to_ids(tokens)
  
          input_mask = [1 if mask_padding_with_zero else 0] * len(
              input_ids
          )
  
          # Zero-pad up to the sequence length.
          padding_length = max_seq_length - len(input_ids)
          if pad_on_left:
              input_ids = ([pad_token] * padding_length) + input_ids
              input_mask = (
                  [0 if mask_padding_with_zero else 1] * padding_length
              ) + input_mask
              segment_ids = (
                  [pad_token_segment_id] * padding_length
              ) + segment_ids
              if include_labels:
                  labels = ([label_pad] * padding_length) + labels
              valid_tokens = ([0] * padding_length) + valid_tokens
          else:
              input_ids = input_ids + ([pad_token] * padding_length)
              input_mask = input_mask + (
                  [0 if mask_padding_with_zero else 1] * padding_length
              )
              segment_ids = segment_ids + (
                  [pad_token_segment_id] * padding_length
              )
              if include_labels:
                  labels = labels + ([label_pad] * padding_length)
              valid_tokens = valid_tokens + ([0] * padding_length)
  
          assert len(input_ids) == max_seq_length
          assert len(input_mask) == max_seq_length
          assert len(segment_ids) == max_seq_length
          assert len(valid_tokens) == max_seq_length
          if include_labels:
              assert len(labels) == max_seq_length
  
          features.append(
              InputFeatures(
                  input_ids=input_ids,
                  input_mask=input_mask,
                  segment_ids=segment_ids,
                  label_id=labels,
                  valid_ids=valid_tokens,
              )
          )
      return features
  
imports: import logging
  from typing import List, Union
  import torch
  from torch.nn import CrossEntropyLoss, Dropout, Linear
  from torch.nn import functional
  from torch.utils.data import DataLoader, SequentialSampler, TensorDataset
  from transformers import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, BertForTokenClassification, BertPreTrainedModel, RobertaConfig, RobertaModel, XLNetModel, XLNetPreTrainedModel
  from nlp_architect.data.sequential_tagging import TokenClsInputExample
  from nlp_architect.models.transformers.base_model import InputFeatures, TransformerBase
  from nlp_architect.models.transformers.quantized_bert import QuantizedBertForTokenClassification
  from nlp_architect.utils.metrics import tagging
label_desc: Use image node_id 7 for calling the TransformerTokenClassifier obj's underlying member method code with example usage: obj._convert_examples_to_features(examples, max_seq_length, tokenizer, include_labels, cls_token_at_end, pad_on_left, cls_token, sep_token, pad_token, sequence_segment_id, sep_token_extra, cls_token_segment_id, pad_token_segment_id, mask_padding_with_zero) and returns: features