code: def tokenize_context(context, encoded_output, config):
      """Tokenize the context corresponding to a single sequence of text"""
      # in the edge case where the chunk is just a single end token, we don't need to alter our context chunk
      seq_len = len(encoded_output.token_ids)
      context_keys = list(
          k for k in sorted(context[0].keys()) if k not in INFO_KEYS
      )
      context_by_char_loc = sorted(
          [
              (c["end"], [c[k] for k in context_keys], c.get("text"))
              for c in context
          ],
          key=lambda c: c[0],
      )
      # default context is set by user in config
      default_context = [
          config.default_context[k] for k in context_keys
      ]
      current_char_loc = 0
      tokenized_context = []
      assert len(encoded_output.tokens) == len(
          encoded_output.token_ends
      )
      if config.include_bos_eos:
          assert (
              encoded_output.token_starts[1]
              <= encoded_output.token_ends[-2]
          )
      else:
          assert (
              encoded_output.token_starts[0]
              <= encoded_output.token_ends[-1]
          )
      for i, (token, char_loc) in enumerate(
          zip(encoded_output.tokens, encoded_output.token_ends)
      ):
          # Note: this assumes that the tokenization will never lump multiple tokens into one
          # (this would not be the case if multiple context spans make up the same token)
          if char_loc == -1:
              tokenized_context.append(default_context)
          else:
              while (
                  token.strip()
                  and char_loc
                  > context_by_char_loc[current_char_loc][0]
              ):
                  current_char_loc += 1
                  if current_char_loc >= len(context_by_char_loc):
                      raise ValueError(
                          "Context cannot be fully matched as it appears to not cover the end of the sequence for token {}".format(
                              token
                          )
                      )
              if (
                  context_by_char_loc[current_char_loc][2]
                  and token.strip().strip("Â")
                  not in context_by_char_loc[current_char_loc][2]
              ):
                  warnings.warn(
                      "subtoken: {} has matched up with the context for: {}".format(
                          repr(token),
                          repr(
                              context_by_char_loc[current_char_loc][2]
                          ),
                      )
                  )
              tokenized_context.append(
                  context_by_char_loc[current_char_loc][1]
              )
  
      assert len(tokenized_context) == len(encoded_output.token_ends)
      # padded value doesn't matter since it will be masked out
      expanded_context = np.pad(
          tokenized_context,
          ((0, seq_len - len(tokenized_context)), (0, 0)),
          "constant",
      )
      assert len(expanded_context) == len(encoded_output.token_ids)
      return expanded_context
  
imports: import warnings
  import functools
  from collections import namedtuple, Counter, OrderedDict
  import spacy
  import numpy
label_desc: Use image node_id 5 for calling a global function with example usage: tokenize_context(context, encoded_output, config) and returns: expanded_context