class_docstr: Vocab embedding implementing the adaptive softmax.
  
  The adaptive softmax was first introduced in this paper
  (https://arxiv.org/abs/1609.04309). Note that this implementation is actually
  most similar to the adaptive vocab embeddings in
  https://arxiv.org/abs/1809.10853 as it supports having different embedding
  sizes for different clusters.
  
  The adaptive softmax works by factorizing the traditional softmax over
  multiple clusters:
    p(v|h) = p(v|c,h) p(c|h),
  where both probability distributions take the form of a softmax.
  
  Further speed up is achieved by putting the class containing the most
  frequently occurring tokens in the "head" cluster. Essentially, those tokens
  are included as "classes" in the p(c|h) softmax. Thus computing their
  probabilities requires only single softmax evaluation.
  
  This implementation differs from vocab_embeddings.AdaptiveVocabEmbedding. That
  implementation only supports variable embeddings sizes across clusters. This
  implementation also supports the adaptive softmax.
  
  A few conditions must be met in order to use this vocab:
    - Unitransformer.shared_embedding_and_softmax_weights = True.
    - If training, then
      Unitranformer.loss_fn = adaptive_softmax.adaptive_softmax_loss_fn.
    - Label smoothing is not supported and will be ignored silently.
    - loss_on_targets_only is not supported and will be ignored silently.
class_docstr_tok: ["Vocab","embedding","implementing","the","adaptive","softmax",".","The","adaptive","softmax","was","first","introduced","in","this","paper","(","https",":","\/\/arxiv.org\/abs\/1609.04309",")",".","Note","that","this","implementation","is","actually","most","similar","to","the","adaptive","vocab","embeddings","in","https",":","\/\/arxiv.org\/abs\/1809.10853","as","it","supports","having","different","embedding","sizes","for","different","clusters",".","The","adaptive","softmax","works","by","factorizing","the","traditional","softmax","over","multiple","clusters",":","p","(","v|h",")","=","p","(","v|c",",","h",")","p","(","c|h",")",",","where","both","probability","distributions","take","the","form","of","a","softmax",".","Further","speed","up","is","achieved","by","putting","the","class","containing","the","most","frequently","occurring","tokens","in","the","``","head","''","cluster",".","Essentially",",","those","tokens","are","included","as","``","classes","''","in","the","p","(","c|h",")","softmax",".","Thus","computing","their","probabilities","requires","only","single","softmax","evaluation",".","This","implementation","differs","from","vocab_embeddings.AdaptiveVocabEmbedding",".","That","implementation","only","supports","variable","embeddings","sizes","across","clusters",".","This","implementation","also","supports","the","adaptive","softmax",".","A","few","conditions","must","be","met","in","order","to","use","this","vocab",":","-","Unitransformer.shared_embedding_and_softmax_weights","=","True",".","-","If","training",",","then","Unitranformer.loss_fn","=","adaptive_softmax.adaptive_softmax_loss_fn",".","-","Label","smoothing","is","not","supported","and","will","be","ignored","silently",".","-","loss_on_targets_only","is","not","supported","and","will","be","ignored","silently","."]
docstr: Creates a cluster given its spec.
docstr_tok: ["Creates","a","cluster","given","its","spec","."]
code: def _create_cluster(self, cluster_spec, index, start_token_id):
      """Creates a cluster given its spec."""
      token_count = cluster_spec["token_count"]
      embedding_size = cluster_spec.get(
          "embedding_size", self._output_dim.size
      )
      length_projection_factor = cluster_spec.get(
          "length_projection_factor", 1
      )
      if length_projection_factor <= 0 or length_projection_factor > 1:
          raise ValueError(
              "Invalid length_projection_factor of {}. Must be in range (0, 1]".format(
                  length_projection_factor
              )
          )
  
      if index == 0:
          # Include the entries for the tail clusters in the head cluster "vocab".
          cluster_vocab_dim = mtf.Dimension(
              self._vocab_dim.name, token_count + self._num_clusters - 1
          )
      else:
          cluster_vocab_dim = mtf.Dimension(
              self._vocab_dim.name, token_count
          )
  
      if embedding_size == self._output_dim.size:
          # In this case we don't need to up project from the embedding space to
          # the model state space.
          cluster_embedding = transformer.VocabEmbedding(
              mesh=self._mesh,
              vocab_dim=cluster_vocab_dim,
              output_dim=self._output_dim,
              variable_dtype=self._variable_dtype,
              name="{}_{}".format(self._name, index),
              ensemble_dim=self._ensemble_dim,
          )
      else:
          cluster_embedding = vocab_embeddings.FactorizedVocabEmbedding(
              mesh=self._mesh,
              vocab_dim=cluster_vocab_dim,
              output_dim=self._output_dim,
              variable_dtype=self._variable_dtype,
              name="{}_{}".format(self._name, index),
              ensemble_dim=self._ensemble_dim,
              inner_dimension_size=embedding_size,
          )
      return _Cluster(
          embedding=cluster_embedding,
          start_token_id=start_token_id,
          end_token_id=start_token_id + token_count,
          length_projection_factor=length_projection_factor,
          vocab_dim=cluster_vocab_dim,
      )
  
imports: import math
  from typing import Dict, Sequence, Union
  import gin
  import mesh_tensorflow
  from mesh_tensorflow.transformer import transformer
  from mesh_tensorflow.transformer import vocab_embeddings
  import tensorflow.compat.v1
  from tensorflow.compat.v1 import estimator
label_desc: Use image node_id 2 for calling the AdaptiveSoftmaxVocabEmbedding obj's underlying member method code with example usage: obj._create_cluster(cluster_spec, index, start_token_id) and returns: _Cluster