class_docstr: Vocabulary class that loads vocabulary from file, and maintains mapping
  tables between token strings and indexes.
  
  Each line of the vocab file should contains one vocabulary token, e.g.,::
  
      vocab_token_1
      vocab token 2
      vocab       token | 3 .
      ...
  
  Args:
      filename (str): Path to the vocabulary file where each line contains
          one token.
      bos_token (str): A special token that will be added to the beginning of
          sequences.
      eos_token (str): A special token that will be added to the end of
          sequences.
      unk_token (str): A special token that will replace all unknown tokens
          (tokens not included in the vocabulary).
      pad_token (str): A special token that is used to do padding.
class_docstr_tok: ["Vocabulary","class","that","loads","vocabulary","from","file",",","and","maintains","mapping","tables","between","token","strings","and","indexes",".","Each","line","of","the","vocab","file","should","contains","one","vocabulary","token",",","e.g.",",",":",":","vocab_token_1","vocab","token","2","vocab","token","|","3",".","...","Args",":","filename","(","str",")",":","Path","to","the","vocabulary","file","where","each","line","contains","one","token",".","bos_token","(","str",")",":","A","special","token","that","will","be","added","to","the","beginning","of","sequences",".","eos_token","(","str",")",":","A","special","token","that","will","be","added","to","the","end","of","sequences",".","unk_token","(","str",")",":","A","special","token","that","will","replace","all","unknown","tokens","(","tokens","not","included","in","the","vocabulary",")",".","pad_token","(","str",")",":","A","special","token","that","is","used","to","do","padding","."]
docstr: Maps text tokens into ids.
  
  The returned ids are a Tensor.
  
  Args:
      tokens: An tensor of text tokens.
  
  Returns:
      A tensor of token ids of the same shape.
docstr_tok: ["Maps","text","tokens","into","ids",".","The","returned","ids","are","a","Tensor",".","Args",":","tokens",":","An","tensor","of","text","tokens",".","Returns",":","A","tensor","of","token","ids","of","the","same","shape","."]
code: def map_tokens_to_ids(self, tokens):
      """Maps text tokens into ids.
  
      The returned ids are a Tensor.
  
      Args:
          tokens: An tensor of text tokens.
  
      Returns:
          A tensor of token ids of the same shape.
      """
      return self.token_to_id_map.lookup(tokens)
  
imports: import warnings
  from collections import defaultdict
  import tensorflow
  from tensorflow import gfile
  import numpy
  from texar.tf.utils.utils import dict_lookup
label_desc: Use image node_id 4 for calling the Vocab obj's underlying member method code with example usage: obj.map_tokens_to_ids(tokens) and returns: self