docstr: Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
  when overlapping namespaces have same padding behaviour (padded/non-padded)
  Summary of namespace paddings in this test:
  original_vocab namespaces
      tokens0     padded
      tokens1     non-padded
      tokens2     padded
      tokens3     non-padded
  instances namespaces
      tokens0     padded
      tokens1     non-padded
      tokens4     padded
      tokens5     non-padded
  TypicalExtention example: (of tokens1 namespace)
  -> original_vocab index2token
     apple          #0->apple
     bat            #1->bat
     cat            #2->cat
  -> Token to be extended with: cat, an, apple, banana, atom, bat
  -> extended_vocab: index2token
     apple           #0->apple
     bat             #1->bat
     cat             #2->cat
     an              #3->an
     atom            #4->atom
     banana          #5->banana
docstr_tok: ["Tests","for","Valid","Vocab","Extension","thoroughly",":","Vocab","extension","is","valid","when","overlapping","namespaces","have","same","padding","behaviour","(","padded\/non-padded",")","Summary","of","namespace","paddings","in","this","test",":","original_vocab","namespaces","tokens0","padded","tokens1","non-padded","tokens2","padded","tokens3","non-padded","instances","namespaces","tokens0","padded","tokens1","non-padded","tokens4","padded","tokens5","non-padded","TypicalExtention","example",":","(","of","tokens1","namespace",")","-",">","original_vocab","index2token","apple","#","0-",">","apple","bat","#","1-",">","bat","cat","#","2-",">","cat","-",">","Token","to","be","extended","with",":","cat",",","an",",","apple",",","banana",",","atom",",","bat","-",">","extended_vocab",":","index2token","apple","#","0-",">","apple","bat","#","1-",">","bat","cat","#","2-",">","cat","an","#","3-",">","an","atom","#","4-",">","atom","banana","#","5-",">","banana"]
code: def test_from_params_valid_vocab_extension_thoroughly(self):
      """
      Tests for Valid Vocab Extension thoroughly: Vocab extension is valid
      when overlapping namespaces have same padding behaviour (padded/non-padded)
      Summary of namespace paddings in this test:
      original_vocab namespaces
          tokens0     padded
          tokens1     non-padded
          tokens2     padded
          tokens3     non-padded
      instances namespaces
          tokens0     padded
          tokens1     non-padded
          tokens4     padded
          tokens5     non-padded
      TypicalExtention example: (of tokens1 namespace)
      -> original_vocab index2token
         apple          #0->apple
         bat            #1->bat
         cat            #2->cat
      -> Token to be extended with: cat, an, apple, banana, atom, bat
      -> extended_vocab: index2token
         apple           #0->apple
         bat             #1->bat
         cat             #2->cat
         an              #3->an
         atom            #4->atom
         banana          #5->banana
      """
  
      vocab_dir = self.TEST_DIR / "vocab_save"
      original_vocab = Vocabulary(
          non_padded_namespaces=["tokens1", "tokens3"]
      )
      original_vocab.add_token_to_namespace(
          "apple", namespace="tokens0"
      )  # index:2
      original_vocab.add_token_to_namespace(
          "bat", namespace="tokens0"
      )  # index:3
      original_vocab.add_token_to_namespace(
          "cat", namespace="tokens0"
      )  # index:4
  
      original_vocab.add_token_to_namespace(
          "apple", namespace="tokens1"
      )  # index:0
      original_vocab.add_token_to_namespace(
          "bat", namespace="tokens1"
      )  # index:1
      original_vocab.add_token_to_namespace(
          "cat", namespace="tokens1"
      )  # index:2
  
      original_vocab.add_token_to_namespace(
          "a", namespace="tokens2"
      )  # index:0
      original_vocab.add_token_to_namespace(
          "b", namespace="tokens2"
      )  # index:1
      original_vocab.add_token_to_namespace(
          "c", namespace="tokens2"
      )  # index:2
  
      original_vocab.add_token_to_namespace(
          "p", namespace="tokens3"
      )  # index:0
      original_vocab.add_token_to_namespace(
          "q", namespace="tokens3"
      )  # index:1
  
      original_vocab.save_to_files(vocab_dir)
  
      text_field0 = TextField(
          [
              Token(t)
              for t in ["cat", "an", "apple", "banana", "atom", "bat"]
          ],
          {"tokens0": SingleIdTokenIndexer("tokens0")},
      )
      text_field1 = TextField(
          [
              Token(t)
              for t in ["cat", "an", "apple", "banana", "atom", "bat"]
          ],
          {"tokens1": SingleIdTokenIndexer("tokens1")},
      )
      text_field4 = TextField(
          [Token(t) for t in ["l", "m", "n", "o"]],
          {"tokens4": SingleIdTokenIndexer("tokens4")},
      )
      text_field5 = TextField(
          [Token(t) for t in ["x", "y", "z"]],
          {"tokens5": SingleIdTokenIndexer("tokens5")},
      )
      instances = Batch(
          [
              Instance(
                  {
                      "text0": text_field0,
                      "text1": text_field1,
                      "text4": text_field4,
                      "text5": text_field5,
                  }
              )
          ]
      )
  
      params = Params(
          {
              "type": "extend",
              "directory": vocab_dir,
              "non_padded_namespaces": ["tokens1", "tokens5"],
          }
      )
      extended_vocab = Vocabulary.from_params(
          params, instances=instances
      )
  
      # namespaces: tokens0, tokens1 is common.
      # tokens2, tokens3 only vocab has. tokens4, tokens5 only instances
      extended_namespaces = {*extended_vocab._token_to_index}
      assert extended_namespaces == {
          "tokens{}".format(i) for i in range(6)
      }
  
      # # Check that _non_padded_namespaces list is consistent after extension
      assert extended_vocab._non_padded_namespaces == {
          "tokens1",
          "tokens3",
          "tokens5",
      }
  
      # # original_vocab["tokens1"] has 3 tokens, instances of "tokens1" ns has 5 tokens. 2 overlapping
      assert extended_vocab.get_vocab_size("tokens1") == 6
      assert (
          extended_vocab.get_vocab_size("tokens0") == 8
      )  # 2 extra overlapping because padded
  
      # namespace tokens3, tokens4 was only in original_vocab,
      # and its token count should be same in extended_vocab
      assert extended_vocab.get_vocab_size(
          "tokens2"
      ) == original_vocab.get_vocab_size("tokens2")
      assert extended_vocab.get_vocab_size(
          "tokens3"
      ) == original_vocab.get_vocab_size("tokens3")
  
      # namespace tokens2 was only in instances,
      # and its token count should be same in extended_vocab
      assert (
          extended_vocab.get_vocab_size("tokens4") == 6
      )  # l,m,n,o + oov + padding
      assert extended_vocab.get_vocab_size("tokens5") == 3  # x,y,z
  
      # Word2index mapping of all words in all namespaces of original_vocab
      # should be maintained in extended_vocab
      for (
          namespace,
          token2index,
      ) in original_vocab._token_to_index.items():
          for token, _ in token2index.items():
              vocab_index = original_vocab.get_token_index(
                  token, namespace
              )
              extended_vocab_index = extended_vocab.get_token_index(
                  token, namespace
              )
              assert vocab_index == extended_vocab_index
      # And same for Index2Word mapping
      for (
          namespace,
          index2token,
      ) in original_vocab._index_to_token.items():
          for index, _ in index2token.items():
              vocab_token = original_vocab.get_token_from_index(
                  index, namespace
              )
              extended_vocab_token = (
                  extended_vocab.get_token_from_index(index, namespace)
              )
              assert vocab_token == extended_vocab_token
  
imports: import codecs
  import gzip
  import pickle
  import shutil
  import zipfile
  from copy import deepcopy
  import pytest
  from allennlp.common.checks import ConfigurationError
  from allennlp.common.params import Params
  from allennlp.common.testing import AllenNlpTestCase
  from allennlp.data import Instance, Token
  from allennlp.data.batch import Batch
  from allennlp.data.fields import TextField
  from allennlp.data.token_indexers import SingleIdTokenIndexer, TokenCharactersIndexer
  from allennlp.data.tokenizers import CharacterTokenizer
  from allennlp.data.vocabulary import _NamespaceDependentDefaultDict, _read_pretrained_tokens, DEFAULT_OOV_TOKEN, Vocabulary
  from allennlp.modules.token_embedders.embedding import format_embeddings_file_uri
label_desc: Use image node_id 21 for calling the TestVocabulary obj's underlying member method code with example usage: obj.test_from_params_valid_vocab_extension_thoroughly() without return types