class_docstr: Text data processor that reads single set of text files. This can be
  used for, e.g., language models, auto-encoders, etc.
  
  Args:
      hparams: A `dict` or instance of :class:`~texar.tf.HParams` containing
          hyperparameters. See :meth:`default_hparams` for the defaults.
  
  By default, the processor reads raw data files, performs tokenization,
  batching and other pre-processing steps, and results in a TF Dataset
  whose element is a python `dict` including three fields:
  
      - "text":
          A string Tensor of shape `[batch_size, max_time]` containing
          the **raw** text toknes. `max_time` is the length of the longest
          sequence in the batch.
          Short sequences in the batch are padded with **empty string**.
          BOS and EOS tokens are added as per
          :attr:`hparams`. Out-of-vocabulary tokens are **NOT** replaced
          with UNK.
      - "text_ids":
          An `int64` Tensor of shape `[batch_size, max_time]`
          containing the token indexes.
      - "length":
          An `int` Tensor of shape `[batch_size]` containing the
          length of each sequence in the batch (including BOS and
          EOS if added).
  
  If :attr:`'variable_utterance'` is set to `True` in :attr:`hparams`, the
  resulting dataset has elements with four fields:
  
      - "text":
          A string Tensor of shape
          `[batch_size, max_utterance, max_time]`, where *max_utterance* is
          either the maximum number of utterances in each elements of the
          batch, or :attr:`max_utterance_cnt` as specified in :attr:`hparams`.
      - "text_ids":
          An `int64` Tensor of shape
          `[batch_size, max_utterance, max_time]` containing the token
          indexes.
      - "length":
          An `int` Tensor of shape `[batch_size, max_utterance]`
          containing the length of each sequence in the batch.
      - "utterance_cnt":
          An `int` Tensor of shape `[batch_size]` containing
          the number of utterances of each element in the batch.
  
  The above field names can be accessed through :attr:`text_name`,
  :attr:`text_id_name`, :attr:`length_name`, and
  :attr:`utterance_cnt_name`, respectively.
  
  Example:
  
      .. code-block:: python
  
          hparams={
              'dataset': { 'files': 'data.txt', 'vocab_file': 'vocab.txt' },
              'batch_size': 1
          }
          data = MonoTextData(hparams)
          iterator = DataIterator(data)
          batch = iterator.get_next()
  
          iterator.switch_to_dataset(sess) # initializes the dataset
          batch_ = sess.run(batch)
          # batch_ == {
          #    'text': [['<BOS>', 'example', 'sequence', '<EOS>']],
          #    'text_ids': [[1, 5, 10, 2]],
          #    'length': [4]
          # }
class_docstr_tok: ["Text","data","processor","that","reads","single","set","of","text","files",".","This","can","be","used","for",",","e.g.",",","language","models",",","auto-encoders",",","etc",".","Args",":","hparams",":","A","`","dict","`","or","instance","of",":","class",":","`","~texar.tf.HParams","`","containing","hyperparameters",".","See",":","meth",":","`","default_hparams","`","for","the","defaults",".","By","default",",","the","processor","reads","raw","data","files",",","performs","tokenization",",","batching","and","other","pre-processing","steps",",","and","results","in","a","TF","Dataset","whose","element","is","a","python","`","dict","`","including","three","fields",":","-","``","text","''",":","A","string","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","*","*","raw","*","*","text","toknes",".","`","max_time","`","is","the","length","of","the","longest","sequence","in","the","batch",".","Short","sequences","in","the","batch","are","padded","with","*","*","empty","string","*","*",".","BOS","and","EOS","tokens","are","added","as","per",":","attr",":","`","hparams","`",".","Out-of-vocabulary","tokens","are","*","*","NOT","*","*","replaced","with","UNK",".","-","``","text_ids","''",":","An","`","int64","`","Tensor","of","shape","`","[","batch_size",",","max_time","]","`","containing","the","token","indexes",".","-","``","length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","containing","the","length","of","each","sequence","in","the","batch","(","including","BOS","and","EOS","if","added",")",".","If",":","attr",":","`","'variable_utterance","'","`","is","set","to","`","True","`","in",":","attr",":","`","hparams","`",",","the","resulting","dataset","has","elements","with","four","fields",":","-","``","text","''",":","A","string","Tensor","of","shape","`","[","batch_size",",","max_utterance",",","max_time","]","`",",","where","*","max_utterance","*","is","either","the","maximum","number","of","utterances","in","each","elements","of","the","batch",",","or",":","attr",":","`","max_utterance_cnt","`","as","specified","in",":","attr",":","`","hparams","`",".","-","``","text_ids","''",":","An","`","int64","`","Tensor","of","shape","`","[","batch_size",",","max_utterance",",","max_time","]","`","containing","the","token","indexes",".","-","``","length","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size",",","max_utterance","]","`","containing","the","length","of","each","sequence","in","the","batch",".","-","``","utterance_cnt","''",":","An","`","int","`","Tensor","of","shape","`","[","batch_size","]","`","containing","the","number","of","utterances","of","each","element","in","the","batch",".","The","above","field","names","can","be","accessed","through",":","attr",":","`","text_name","`",",",":","attr",":","`","text_id_name","`",",",":","attr",":","`","length_name","`",",","and",":","attr",":","`","utterance_cnt_name","`",",","respectively",".","Example",":","..","code-block",":",":","python","hparams=","{","'dataset","'",":","{","'files","'",":","'data.txt","'",",","'vocab_file","'",":","'vocab.txt","'","}",",","'batch_size","'",":","1","}","data","=","MonoTextData","(","hparams",")","iterator","=","DataIterator","(","data",")","batch","=","iterator.get_next","(",")","iterator.switch_to_dataset","(","sess",")","#","initializes","the","dataset","batch_","=","sess.run","(","batch",")","#","batch_","==","{","#","'text","'",":","[","[","'","<","BOS",">","'",",","'example","'",",","'sequence","'",",","'","<","EOS",">","'","]","]",",","#","'text_ids","'",":","[","[","1",",","5",",","10",",","2","]","]",",","#","'length","'",":","[","4","]","#","}"]
docstr: Returns a dicitionary of default hyperparameters:
  
  .. code-block:: python
  
      {
          # (1) Hyperparams specific to text dataset
          "dataset": {
              "files": [],
              "compression_type": None,
              "vocab_file": "",
              "embedding_init": {},
              "delimiter": " ",
              "max_seq_length": None,
              "length_filter_mode": "truncate",
              "pad_to_max_seq_length": False,
              "bos_token": "<BOS>"
              "eos_token": "<EOS>"
              "other_transformations": [],
              "variable_utterance": False,
              "utterance_delimiter": "|||",
              "max_utterance_cnt": 5,
              "data_name": None,
          }
          # (2) General hyperparams
          "num_epochs": 1,
          "batch_size": 64,
          "allow_smaller_final_batch": True,
          "shuffle": True,
          "shuffle_buffer_size": None,
          "shard_and_shuffle": False,
          "num_parallel_calls": 1,
          "prefetch_buffer_size": 0,
          "max_dataset_size": -1,
          "seed": None,
          "name": "mono_text_data",
          # (3) Bucketing
          "bucket_boundaries": [],
          "bucket_batch_sizes": None,
          "bucket_length_fn": None,
      }
  
  Here:
  
  1. For the hyperparameters in the :attr:`"dataset"` field:
  
      "files": str or list
          A (list of) text file path(s).
  
          Each line contains a single text sequence.
  
      "compression_type": str, optional
          One of "" (no compression), "ZLIB", or "GZIP".
  
      "vocab_file": str
          Path to vocabulary file. Each line of the file should contain
          one vocabulary token.
  
          Used to create an instance of :class:`~texar.tf.data.Vocab`.
  
      "embedding_init": dict
          The hyperparameters for pre-trained embedding loading and
          initialization.
  
          The structure and default values are defined in
          :meth:`texar.tf.data.Embedding.default_hparams`.
  
      "delimiter": str
          The delimiter to split each line of the text files into tokens.
  
      "max_seq_length": int, optional
          Maximum length of output sequences. Data samples exceeding the
          length will be truncated or discarded according to
          :attr:`"length_filter_mode"`. The length does not include
          any added
          :attr:`"bos_token"` or :attr:`"eos_token"`. If `None` (default),
          no filtering is performed.
  
      "length_filter_mode": str
          Either "truncate" or "discard". If "truncate" (default),
          tokens exceeding the :attr:`"max_seq_length"` will be truncated.
          If "discard", data samples longer than the
          :attr:`"max_seq_length"`
          will be discarded.
  
      "pad_to_max_seq_length": bool
          If `True`, pad all data instances to length
          :attr:`"max_seq_length"`.
          Raises error if :attr:`"max_seq_length"` is not provided.
  
      "bos_token": str
          The Begin-Of-Sequence token prepended to each sequence.
  
          Set to an empty string to avoid prepending.
  
      "eos_token": str
          The End-Of-Sequence token appended to each sequence.
  
          Set to an empty string to avoid appending.
  
      "other_transformations": list
          A list of transformation functions or function names/paths to
          further transform each single data instance.
  
          (More documentations to be added.)
  
      "variable_utterance": bool
          If `True`, each line of the text file is considered to contain
          multiple sequences (utterances) separated by
          :attr:`"utterance_delimiter"`.
  
          For example, in dialog data, each line can contain a series of
          dialog history utterances. See the example in
          `examples/hierarchical_dialog` for a use case.
  
      "utterance_delimiter": str
          The delimiter to split over utterance level. Should not be the
          same with :attr:`"delimiter"`. Used only when
          :attr:`"variable_utterance"``==True`.
  
      "max_utterance_cnt": int
          Maximally allowed number of utterances in a data instance.
          Extra utterances are truncated out.
  
      "data_name": str
          Name of the dataset.
  
  2. For the **general** hyperparameters, see
  :meth:`texar.tf.data.DataBase.default_hparams` for details.
  
  3. **Bucketing** is to group elements of the dataset together by length
  and then pad and batch. (See more at
  :tf_main:`bucket_by_sequence_length
  <contrib/data/bucket_by_sequence_length>`). For bucketing
  hyperparameters:
  
      "bucket_boundaries": list
          An int list containing the upper length boundaries of the
          buckets.
  
          Set to an empty list (default) to disable bucketing.
  
      "bucket_batch_sizes": list
          An int list containing batch size per bucket. Length should be
          `len(bucket_boundaries) + 1`.
  
          If `None`, every bucket whill have the same batch size specified
          in :attr:`batch_size`.
  
      "bucket_length_fn": str or callable
          Function maps dataset element to `tf.int32` scalar, determines
          the length of the element.
  
          This can be a function, or the name or full module path to the
          function. If function name is given, the function must be in the
          :mod:`texar.tf.custom` module.
  
          If `None` (default), length is determined by the number of
          tokens (including BOS and EOS if added) of the element.
docstr_tok: ["Returns","a","dicitionary","of","default","hyperparameters",":","..","code-block",":",":","python","{","#","(","1",")","Hyperparams","specific","to","text","dataset","``","dataset","''",":","{","``","files","''",":","[","]",",","``","compression_type","''",":","None",",","``","vocab_file","''",":","``","''",",","``","embedding_init","''",":","{","}",",","``","delimiter","''",":","``","``",",","``","max_seq_length","''",":","None",",","``","length_filter_mode","''",":","``","truncate","''",",","``","pad_to_max_seq_length","''",":","False",",","``","bos_token","''",":","``","<","BOS",">","''","``","eos_token","''",":","``","<","EOS",">","''","``","other_transformations","''",":","[","]",",","``","variable_utterance","''",":","False",",","``","utterance_delimiter","''",":","``","|||","''",",","``","max_utterance_cnt","''",":","5",",","``","data_name","''",":","None",",","}","#","(","2",")","General","hyperparams","``","num_epochs","''",":","1",",","``","batch_size","''",":","64",",","``","allow_smaller_final_batch","''",":","True",",","``","shuffle","''",":","True",",","``","shuffle_buffer_size","''",":","None",",","``","shard_and_shuffle","''",":","False",",","``","num_parallel_calls","''",":","1",",","``","prefetch_buffer_size","''",":","0",",","``","max_dataset_size","''",":","-1",",","``","seed","''",":","None",",","``","name","''",":","``","mono_text_data","''",",","#","(","3",")","Bucketing","``","bucket_boundaries","''",":","[","]",",","``","bucket_batch_sizes","''",":","None",",","``","bucket_length_fn","''",":","None",",","}","Here",":","1",".","For","the","hyperparameters","in","the",":","attr",":","`","``","dataset","''","`","field",":","``","files","''",":","str","or","list","A","(","list","of",")","text","file","path","(","s",")",".","Each","line","contains","a","single","text","sequence",".","``","compression_type","''",":","str",",","optional","One","of","``","''","(","no","compression",")",",","``","ZLIB","''",",","or","``","GZIP","''",".","``","vocab_file","''",":","str","Path","to","vocabulary","file",".","Each","line","of","the","file","should","contain","one","vocabulary","token",".","Used","to","create","an","instance","of",":","class",":","`","~texar.tf.data.Vocab","`",".","``","embedding_init","''",":","dict","The","hyperparameters","for","pre-trained","embedding","loading","and","initialization",".","The","structure","and","default","values","are","defined","in",":","meth",":","`","texar.tf.data.Embedding.default_hparams","`",".","``","delimiter","''",":","str","The","delimiter","to","split","each","line","of","the","text","files","into","tokens",".","``","max_seq_length","''",":","int",",","optional","Maximum","length","of","output","sequences",".","Data","samples","exceeding","the","length","will","be","truncated","or","discarded","according","to",":","attr",":","`","``","length_filter_mode","''","`",".","The","length","does","not","include","any","added",":","attr",":","`","``","bos_token","''","`","or",":","attr",":","`","``","eos_token","''","`",".","If","`","None","`","(","default",")",",","no","filtering","is","performed",".","``","length_filter_mode","''",":","str","Either","``","truncate","''","or","``","discard","''",".","If","``","truncate","''","(","default",")",",","tokens","exceeding","the",":","attr",":","`","``","max_seq_length","''","`","will","be","truncated",".","If","``","discard","''",",","data","samples","longer","than","the",":","attr",":","`","``","max_seq_length","''","`","will","be","discarded",".","``","pad_to_max_seq_length","''",":","bool","If","`","True","`",",","pad","all","data","instances","to","length",":","attr",":","`","``","max_seq_length","''","`",".","Raises","error","if",":","attr",":","`","``","max_seq_length","''","`","is","not","provided",".","``","bos_token","''",":","str","The","Begin-Of-Sequence","token","prepended","to","each","sequence",".","Set","to","an","empty","string","to","avoid","prepending",".","``","eos_token","''",":","str","The","End-Of-Sequence","token","appended","to","each","sequence",".","Set","to","an","empty","string","to","avoid","appending",".","``","other_transformations","''",":","list","A","list","of","transformation","functions","or","function","names\/paths","to","further","transform","each","single","data","instance",".","(","More","documentations","to","be","added",".",")","``","variable_utterance","''",":","bool","If","`","True","`",",","each","line","of","the","text","file","is","considered","to","contain","multiple","sequences","(","utterances",")","separated","by",":","attr",":","`","``","utterance_delimiter","''","`",".","For","example",",","in","dialog","data",",","each","line","can","contain","a","series","of","dialog","history","utterances",".","See","the","example","in","`","examples\/hierarchical_dialog","`","for","a","use","case",".","``","utterance_delimiter","''",":","str","The","delimiter","to","split","over","utterance","level",".","Should","not","be","the","same","with",":","attr",":","`","``","delimiter","''","`",".","Used","only","when",":","attr",":","`","``","variable_utterance","''","``","==True","`",".","``","max_utterance_cnt","''",":","int","Maximally","allowed","number","of","utterances","in","a","data","instance",".","Extra","utterances","are","truncated","out",".","``","data_name","''",":","str","Name","of","the","dataset",".","2",".","For","the","*","*","general","*","*","hyperparameters",",","see",":","meth",":","`","texar.tf.data.DataBase.default_hparams","`","for","details",".","3",".","*","*","Bucketing","*","*","is","to","group","elements","of","the","dataset","together","by","length","and","then","pad","and","batch",".","(","See","more","at",":","tf_main",":","`","bucket_by_sequence_length","<","contrib\/data\/bucket_by_sequence_length",">","`",")",".","For","bucketing","hyperparameters",":","``","bucket_boundaries","''",":","list","An","int","list","containing","the","upper","length","boundaries","of","the","buckets",".","Set","to","an","empty","list","(","default",")","to","disable","bucketing",".","``","bucket_batch_sizes","''",":","list","An","int","list","containing","batch","size","per","bucket",".","Length","should","be","`","len","(","bucket_boundaries",")","+","1","`",".","If","`","None","`",",","every","bucket","whill","have","the","same","batch","size","specified","in",":","attr",":","`","batch_size","`",".","``","bucket_length_fn","''",":","str","or","callable","Function","maps","dataset","element","to","`","tf.int32","`","scalar",",","determines","the","length","of","the","element",".","This","can","be","a","function",",","or","the","name","or","full","module","path","to","the","function",".","If","function","name","is","given",",","the","function","must","be","in","the",":","mod",":","`","texar.tf.custom","`","module",".","If","`","None","`","(","default",")",",","length","is","determined","by","the","number","of","tokens","(","including","BOS","and","EOS","if","added",")","of","the","element","."]
code: def default_hparams():
      """Returns a dicitionary of default hyperparameters:
  
      .. code-block:: python
  
          {
              # (1) Hyperparams specific to text dataset
              "dataset": {
                  "files": [],
                  "compression_type": None,
                  "vocab_file": "",
                  "embedding_init": {},
                  "delimiter": " ",
                  "max_seq_length": None,
                  "length_filter_mode": "truncate",
                  "pad_to_max_seq_length": False,
                  "bos_token": "<BOS>"
                  "eos_token": "<EOS>"
                  "other_transformations": [],
                  "variable_utterance": False,
                  "utterance_delimiter": "|||",
                  "max_utterance_cnt": 5,
                  "data_name": None,
              }
              # (2) General hyperparams
              "num_epochs": 1,
              "batch_size": 64,
              "allow_smaller_final_batch": True,
              "shuffle": True,
              "shuffle_buffer_size": None,
              "shard_and_shuffle": False,
              "num_parallel_calls": 1,
              "prefetch_buffer_size": 0,
              "max_dataset_size": -1,
              "seed": None,
              "name": "mono_text_data",
              # (3) Bucketing
              "bucket_boundaries": [],
              "bucket_batch_sizes": None,
              "bucket_length_fn": None,
          }
  
      Here:
  
      1. For the hyperparameters in the :attr:`"dataset"` field:
  
          "files": str or list
              A (list of) text file path(s).
  
              Each line contains a single text sequence.
  
          "compression_type": str, optional
              One of "" (no compression), "ZLIB", or "GZIP".
  
          "vocab_file": str
              Path to vocabulary file. Each line of the file should contain
              one vocabulary token.
  
              Used to create an instance of :class:`~texar.tf.data.Vocab`.
  
          "embedding_init": dict
              The hyperparameters for pre-trained embedding loading and
              initialization.
  
              The structure and default values are defined in
              :meth:`texar.tf.data.Embedding.default_hparams`.
  
          "delimiter": str
              The delimiter to split each line of the text files into tokens.
  
          "max_seq_length": int, optional
              Maximum length of output sequences. Data samples exceeding the
              length will be truncated or discarded according to
              :attr:`"length_filter_mode"`. The length does not include
              any added
              :attr:`"bos_token"` or :attr:`"eos_token"`. If `None` (default),
              no filtering is performed.
  
          "length_filter_mode": str
              Either "truncate" or "discard". If "truncate" (default),
              tokens exceeding the :attr:`"max_seq_length"` will be truncated.
              If "discard", data samples longer than the
              :attr:`"max_seq_length"`
              will be discarded.
  
          "pad_to_max_seq_length": bool
              If `True`, pad all data instances to length
              :attr:`"max_seq_length"`.
              Raises error if :attr:`"max_seq_length"` is not provided.
  
          "bos_token": str
              The Begin-Of-Sequence token prepended to each sequence.
  
              Set to an empty string to avoid prepending.
  
          "eos_token": str
              The End-Of-Sequence token appended to each sequence.
  
              Set to an empty string to avoid appending.
  
          "other_transformations": list
              A list of transformation functions or function names/paths to
              further transform each single data instance.
  
              (More documentations to be added.)
  
          "variable_utterance": bool
              If `True`, each line of the text file is considered to contain
              multiple sequences (utterances) separated by
              :attr:`"utterance_delimiter"`.
  
              For example, in dialog data, each line can contain a series of
              dialog history utterances. See the example in
              `examples/hierarchical_dialog` for a use case.
  
          "utterance_delimiter": str
              The delimiter to split over utterance level. Should not be the
              same with :attr:`"delimiter"`. Used only when
              :attr:`"variable_utterance"``==True`.
  
          "max_utterance_cnt": int
              Maximally allowed number of utterances in a data instance.
              Extra utterances are truncated out.
  
          "data_name": str
              Name of the dataset.
  
      2. For the **general** hyperparameters, see
      :meth:`texar.tf.data.DataBase.default_hparams` for details.
  
      3. **Bucketing** is to group elements of the dataset together by length
      and then pad and batch. (See more at
      :tf_main:`bucket_by_sequence_length
      <contrib/data/bucket_by_sequence_length>`). For bucketing
      hyperparameters:
  
          "bucket_boundaries": list
              An int list containing the upper length boundaries of the
              buckets.
  
              Set to an empty list (default) to disable bucketing.
  
          "bucket_batch_sizes": list
              An int list containing batch size per bucket. Length should be
              `len(bucket_boundaries) + 1`.
  
              If `None`, every bucket whill have the same batch size specified
              in :attr:`batch_size`.
  
          "bucket_length_fn": str or callable
              Function maps dataset element to `tf.int32` scalar, determines
              the length of the element.
  
              This can be a function, or the name or full module path to the
              function. If function name is given, the function must be in the
              :mod:`texar.tf.custom` module.
  
              If `None` (default), length is determined by the number of
              tokens (including BOS and EOS if added) of the element.
  
      """
      hparams = TextDataBase.default_hparams()
      hparams["name"] = "mono_text_data"
      hparams.update({"dataset": _default_mono_text_dataset_hparams()})
      return hparams
  
imports: import tensorflow
  from texar.tf.utils import utils
  from texar.tf.utils.dtypes import is_callable
  from texar.tf.data.data_utils import count_file_lines
  from texar.tf.data.data import dataset_utils
  from texar.tf.data.data.text_data_base import TextDataBase
  from texar.tf.data.data_decoders import TextDataDecoder, VarUttTextDataDecoder
  from texar.tf.data.vocabulary import Vocab, SpecialTokens
  from texar.tf.data.embedding import Embedding
label_desc: Use image node_id 2 for calling the MonoTextData obj's underlying member method code with example usage: obj.default_hparams() and returns: hparams