class_docstr: Text data processor that reads single set of text files. This can be
  used for, e.g., language models, auto-encoders, etc.
  
  Args:
      hparams: A `dict` or instance of :class:`~texar.tf.HParams` containing
          hyperparameters. See :meth:`default_hparams` for the defaults.
  
  By default, the processor reads raw data files, performs tokenization,
  batching and other pre-processing steps, and results in a TF Dataset
  whose element is a python `dict` including three fields:
  
      - "text":
          A string Tensor of shape `[batch_size, max_time]` containing
          the **raw** text toknes. `max_time` is the length of the longest
          sequence in the batch.
          Short sequences in the batch are padded with **empty string**.
          BOS and EOS tokens are added as per
          :attr:`hparams`. Out-of-vocabulary tokens are **NOT** replaced
          with UNK.
      - "text_ids":
          An `int64` Tensor of shape `[batch_size, max_time]`
          containing the token indexes.
      - "length":
          An `int` Tensor of shape `[batch_size]` containing the
          length of each sequence in the batch (including BOS and
          EOS if added).
  
  If :attr:`'variable_utterance'` is set to `True` in :attr:`hparams`, the
  resulting dataset has elements with four fields:
  
      - "text":
          A string Tensor of shape
          `[batch_size, max_utterance, max_time]`, where *max_utterance* is
          either the maximum number of utterances in each elements of the
          batch, or :attr:`max_utterance_cnt` as specified in :attr:`hparams`.
      - "text_ids":
          An `int64` Tensor of shape
          `[batch_size, max_utterance, max_time]` containing the token
          indexes.
      - "length":
          An `int` Tensor of shape `[batch_size, max_utterance]`
          containing the length of each sequence in the batch.
      - "utterance_cnt":
          An `int` Tensor of shape `[batch_size]` containing
          the number of utterances of each element in the batch.
  
  The above field names can be accessed through :attr:`text_name`,
  :attr:`text_id_name`, :attr:`length_name`, and
  :attr:`utterance_cnt_name`, respectively.
  
  Example:
  
      .. code-block:: python
  
          hparams={
              'dataset': { 'files': 'data.txt', 'vocab_file': 'vocab.txt' },
              'batch_size': 1
          }
          data = MonoTextData(hparams)
          iterator = DataIterator(data)
          batch = iterator.get_next()
  
          iterator.switch_to_dataset(sess) # initializes the dataset
          batch_ = sess.run(batch)
          # batch_ == {
          #    'text': [['<BOS>', 'example', 'sequence', '<EOS>']],
          #    'text_ids': [[1, 5, 10, 2]],
          #    'length': [4]
          # }
class_docstr_tok: ["Text","data","processor","that","reads","single","set","of","text","files",".","This","can","be","used","for",",","e.g.",",","language","models",",","auto-encoders",",","etc",".","Args",":","hparams",":","A","`","dict","`","or","instance","of",":","class",":","`","~texar.tf.HParams","`","containing","hyperparameters",".","See",":","meth",":","`","default_hparams","`","for","the","defaults",".","By","default",",","the","processor","reads","raw","data","files",",","performs","tokenization",",","batching","and","other","pre-processing","steps",",","and","results","in","a","TF","Dataset","whose","element","is","a","python","`","dict","`","including","three","fields",":","-","``","text","''",":","A","string","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","*","*","raw","*","*","text","toknes",".","`","max_time","`","is","the","length","of","the","longest","sequence","in","the","batch",".","Short","sequences","in","the","batch","are","padded","with","*","*","empty","string","*","*",".","BOS","and","EOS","tokens","are","added","as","per",":","attr",":","`","hparams","`",".","Out-of-vocabulary","tokens","are","*","*","NOT","*","*","replaced","with","UNK",".","-","``","text_ids","''",":","An","`","int64","`","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","token","indexes",".","-","``","length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","containing","the","length","of","each","sequence","in","the","batch","(","including","BOS","and","EOS","if","added",")",".","If",":","attr",":","`","'variable_utterance","'","`","is","set","to","`","True","`","in",":","attr",":","`","hparams","`",",","the","resulting","dataset","has","elements","with","four","fields",":","-","``","text","''",":","A","string","Tensor","of","shape","`","[","batch_size",",","max_utterance",",","max_time","]","`",",","where","*","max_utterance","*","is","either","the","maximum","number","of","utterances","in","each","elements","of","the","batch",",","or",":","attr",":","`","max_utterance_cnt","`","as","specified","in",":","attr",":","`","hparams","`",".","-","``","text_ids","''",":","An","`","int64","`","Tensor","of","shape","`","[","batch_size",",","max_utterance",",","max_time","]","`","containing","the","token","indexes",".","-","``","length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size",",","max_utterance","]","`","containing","the","length","of","each","sequence","in","the","batch",".","-","``","utterance_cnt","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","containing","the","number","of","utterances","of","each","element","in","the","batch",".","The","above","field","names","can","be","accessed","through",":","attr",":","`","text_name","`",",",":","attr",":","`","text_id_name","`",",",":","attr",":","`","length_name","`",",","and",":","attr",":","`","utterance_cnt_name","`",",","respectively",".","Example",":","..","code-block",":",":","python","hparams=","{","'dataset","'",":","{","'files","'",":","'data.txt","'",",","'vocab_file","'",":","'vocab.txt","'","}",",","'batch_size","'",":","1","}","data","=","MonoTextData","(","hparams",")","iterator","=","DataIterator","(","data",")","batch","=","iterator.get_next","(",")","iterator.switch_to_dataset","(","sess",")","#","initializes","the","dataset","batch_","=","sess.run","(","batch",")","#","batch_","==","{","#","'text","'",":","[","[","'","<","BOS",">","'",",","'example","'",",","'sequence","'",",","'","<","EOS",">","'","]","]",",","#","'text_ids","'",":","[","[","1",",","5",",","10",",","2","]","]",",","#","'length","'",":","[","4","]","#","}"]
docstr: The dataset, an instance of
  :tf_main:`TF dataset <data/TextLineDataset>`.
docstr_tok: ["The","dataset",",","an","instance","of",":","tf_main",":","`","TF","dataset","<","data\/TextLineDataset",">","`","."]
code: def dataset(self):
      """The dataset, an instance of
      :tf_main:`TF dataset <data/TextLineDataset>`.
      """
      return self._dataset
  
imports: import tensorflow
  from texar.tf.utils import utils
  from texar.tf.utils.dtypes import is_callable
  from texar.tf.data.data_utils import count_file_lines
  from texar.tf.data.data import dataset_utils
  from texar.tf.data.data.text_data_base import TextDataBase
  from texar.tf.data.data_decoders import TextDataDecoder, VarUttTextDataDecoder
  from texar.tf.data.vocabulary import Vocab, SpecialTokens
  from texar.tf.data.embedding import Embedding
label_desc: Use image node_id 15 for calling the MonoTextData obj's underlying member method code with example usage: obj.dataset() and returns: self