docstr: We re-tokenize and re-numberize the raw context (doc_str) here to get doc_tokens to get
  access to start_idx and end_idx mappings.  At this point, ans_token_start is the start index
  of the answer within tokens and ans_token_end is the end index. We calculate the offset of doc_tokens
  within tokens.
  Then we find the start_idx and end_idx
  as well as the corresponding span in the raw text using the answer token indices.
docstr_tok: ["We","re-tokenize","and","re-numberize","the","raw","context","(","doc_str",")","here","to","get","doc_tokens","to","get","access","to","start_idx","and","end_idx","mappings",".","At","this","point",",","ans_token_start","is","the","start","index","of","the","answer","within","tokens","and","ans_token_end","is","the","end","index",".","We","calculate","the","offset","of","doc_tokens","within","tokens",".","Then","we","find","the","start_idx","and","end_idx","as","well","as","the","corresponding","span","in","the","raw","text","using","the","answer","token","indices","."]
code: def _unnumberize(
      self, ans_token_start, ans_token_end, tokens, doc_str
  ):
      """
      We re-tokenize and re-numberize the raw context (doc_str) here to get doc_tokens to get
      access to start_idx and end_idx mappings.  At this point, ans_token_start is the start index
      of the answer within tokens and ans_token_end is the end index. We calculate the offset of doc_tokens
      within tokens.
      Then we find the start_idx and end_idx
      as well as the corresponding span in the raw text using the answer token indices.
      """
      # start_idx and end_idx are lists of char start and end positions in doc_str.
      doc_tokens, start_idxs, end_idxs = self.tensorizer._lookup_tokens(
          doc_str, seq_len=self.tensorizer.max_subseq_len
      )
      # find the offset of doc_tokens in tokens
      try:
          offset_end = (
              tokens.index(self.tensorizer.vocab.get_pad_index()) - 1
          )
      except ValueError:
          offset_end = len(tokens) - 1
      offset_start = list(
          map(
              lambda x: tokens[x:offset_end]
              == doc_tokens[: offset_end - x],
              range(offset_end),
          )
      ).index(True)
      assert offset_start > -1
  
      # find the answer char idxs
      start_char_idx = 0
      end_char_idx = end_idxs[-1]
      try:
          start_char_idx = start_idxs[ans_token_start - offset_start]
          end_char_idx = end_idxs[ans_token_end - offset_start]
      except IndexError:
          # if token indices fall outside the bounds due to a model misprediction.
          pass
      if (ans_token_end - offset_start).item() == -offset_start:
          start_char_idx = 0
          end_char_idx = 0
      ans_str = doc_str[start_char_idx:end_char_idx]
      return ans_str, start_char_idx, end_char_idx
  
imports: import re
  import string
  from collections import Counter
  from itertools import zip_longest
  from typing import Dict, List
  import numpy
  from pytext.common.constants import Stage
  from pytext.metric_reporters.channel import Channel, ConsoleChannel, FileChannel
  from pytext.metric_reporters.metric_reporter import MetricReporter
  from pytext.metrics import compute_classification_metrics, LabelPrediction
  from pytext.metrics.squad_metrics import SquadMetrics
label_desc: Use image node_id 15 for calling the SquadMetricReporter obj's underlying member method code with example usage: obj._unnumberize(ans_token_start, ans_token_end, tokens, doc_str) and returns: ans_str,  start_char_idx,  end_char_idx