class_docstr: Class for handling KGE models which follows the ranking based protocol.
  
  Example
  -------
  >>> # create model and compile using user defined optimizer settings and
  >>> # user defined settings of an existing loss
  >>> from ampligraph.datasets import load_fb15k_237
  >>> from ampligraph.latent_features import ScoringBasedEmbeddingModel
  >>> from ampligraph.latent_features.loss_functions import SelfAdversarialLoss
  >>> import tensorflow as tf
  >>> X = load_fb15k_237()
  >>> loss = SelfAdversarialLoss({'margin': 0.1, 'alpha': 5, 'reduction': 'sum'})
  >>> model = ScoringBasedEmbeddingModel(eta=5,
  >>>                                    k=300,
  >>>                                    scoring_type='ComplEx',
  >>>                                    seed=0)
  >>> model.compile(optimizer='adam', loss=loss)
  >>> model.fit(X['train'],
  >>>           batch_size=10000,
  >>>           epochs=5)
  Epoch 1/5
  29/29 [==============================] - 3s 87ms/step - loss: 13496.5752
  Epoch 2/5
  29/29 [==============================] - 1s 36ms/step - loss: 13488.8682
  Epoch 3/5
  29/29 [==============================] - 1s 35ms/step - loss: 13436.2725
  Epoch 4/5
  29/29 [==============================] - 1s 35ms/step - loss: 13259.0840
  Epoch 5/5
  29/29 [==============================] - 1s 34ms/step - loss: 12977.0117
class_docstr_tok: ["Class","for","handling","KGE","models","which","follows","the","ranking","based","protocol",".","Example","--","--","--","-",">",">",">","#","create","model","and","compile","using","user","defined","optimizer","settings","and",">",">",">","#","user","defined","settings","of","an","existing","loss",">",">",">","from","ampligraph.datasets","import","load_fb15k_237",">",">",">","from","ampligraph.latent_features","import","ScoringBasedEmbeddingModel",">",">",">","from","ampligraph.latent_features.loss_functions","import","SelfAdversarialLoss",">",">",">","import","tensorflow","as","tf",">",">",">","X","=","load_fb15k_237","(",")",">",">",">","loss","=","SelfAdversarialLoss","(","{","'margin","'",":","0.1",",","'alpha","'",":","5",",","'reduction","'",":","'sum","'","}",")",">",">",">","model","=","ScoringBasedEmbeddingModel","(","eta=5",",",">",">",">","k=300",",",">",">",">","scoring_type='ComplEx","'",",",">",">",">","seed=0",")",">",">",">","model.compile","(","optimizer='adam","'",",","loss=loss",")",">",">",">","model.fit","(","X","[","'train","'","]",",",">",">",">","batch_size=10000",",",">",">",">","epochs=5",")","Epoch","1\/5","29\/29","[","==============================","]","-","3s","87ms\/step","-","loss",":","13496.5752","Epoch","2\/5","29\/29","[","==============================","]","-","1s","36ms\/step","-","loss",":","13488.8682","Epoch","3\/5","29\/29","[","==============================","]","-","1s","35ms\/step","-","loss",":","13436.2725","Epoch","4\/5","29\/29","[","==============================","]","-","1s","35ms\/step","-","loss",":","13259.0840","Epoch","5\/5","29\/29","[","==============================","]","-","1s","34ms\/step","-","loss",":","12977.0117"]
docstr: Evaluate the inputs against corruptions and return ranks.
  
  Parameters
  ----------
  x: np.array, shape (n,3) or str or GraphDataLoader or AbstractGraphPartitioner
      Data OR Filename of the data file OR Data Handle to be used for training.
  batch_size: int
      Batch size to use during training.
      May be overridden if ``x`` is `GraphDataLoader` or `AbstractGraphPartitioner` instance
  verbose: bool
      Verbosity mode.
  use_filter: bool or dict
      Whether to use a filter of not. If a dictionary is specified, the data in the dict is concatenated
      and used as filter.
  corrupt_side: str
      Which side to corrupt of a triple to corrupt. It can be the subject (``corrupt_size="s"``),
      the object (``corrupt_size="o"``), the subject and the object (``corrupt_size="s+o"`` or
      ``corrupt_size="s,o"``) (default:`"s,o"`).
  ranking_strategy: str
      Indicates how to break ties when a test triple gets the same rank of a corruption.
      Can be one of the three types: `"best"`, `"middle"`, `"worst"` (default: `"worst"`, i.e.,
      the worst rank is assigned to the test triple).
  entities_subset: list or np.array
      Subset of entities to be used for generating corruptions.
  callbacks: list of keras.callbacks.Callback instances
      List of callbacks to apply during evaluation.
  
  Returns
  -------
  rank: np.array, shape (n, number of corrupted sides)
      Ranking of test triples against subject corruptions and/or object corruptions.
  
  Example
  -------
  >>> from ampligraph.datasets import load_fb15k_237
  >>> from ampligraph.latent_features import ScoringBasedEmbeddingModel
  >>> from ampligraph.evaluation.metrics import mrr_score, hits_at_n_score, mr_score
  >>> X = load_fb15k_237()
  >>> model = ScoringBasedEmbeddingModel(eta=5,
  >>>                                    k=300,
  >>>                                    scoring_type='ComplEx',
  >>>                                    seed=0)
  >>> model.compile(optimizer='adam', loss='nll')
  >>> model.fit(X['train'],
  >>>           batch_size=10000,
  >>>           epochs=5)
  Epoch 1/5
  29/29 [==============================] - 2s 71ms/step - loss: 67361.3047
  Epoch 2/5
  29/29 [==============================] - 1s 35ms/step - loss: 67318.6094
  Epoch 3/5
  29/29 [==============================] - 1s 35ms/step - loss: 67020.0703
  Epoch 4/5
  29/29 [==============================] - 1s 33ms/step - loss: 65867.3750
  Epoch 5/5
  29/29 [==============================] - 1s 34ms/step - loss: 63517.9062
  >>> ranks = model.evaluate(X['test'],
  >>>                        batch_size=100,
  >>>                        corrupt_side='s,o',
  >>>                        use_filter={'train': X['train'],
  >>>                                    'valid': X['valid'],
  >>>                                    'test': X['test'])
  >>> mr_score(ranks), mrr_score(ranks), hits_at_n_score(ranks, 1), hits_at_n_score(ranks, 10), len(ranks)
  28 triples containing invalid keys skipped!
  9 triples containing invalid keys skipped!
  2045/2045 [==============================] - 149s 73ms/step
  (428.44671689989235,
   0.25761041025282316,
   0.1898179861043155,
   0.391965945787259,
   20438)
docstr_tok: ["Evaluate","the","inputs","against","corruptions","and","return","ranks",".","Parameters","--","--","--","--","--","x",":","np.array",",","shape","(","n,3",")","or","str","or","GraphDataLoader","or","AbstractGraphPartitioner","Data","OR","Filename","of","the","data","file","OR","Data","Handle","to","be","used","for","training",".","batch_size",":","int","Batch","size","to","use","during","training",".","May","be","overridden","if","``","x","``","is","`","GraphDataLoader","`","or","`","AbstractGraphPartitioner","`","instance","verbose",":","bool","Verbosity","mode",".","use_filter",":","bool","or","dict","Whether","to","use","a","filter","of","not",".","If","a","dictionary","is","specified",",","the","data","in","the","dict","is","concatenated","and","used","as","filter",".","corrupt_side",":","str","Which","side","to","corrupt","of","a","triple","to","corrupt",".","It","can","be","the","subject","(","``","corrupt_size=","''","s","''","``",")",",","the","object","(","``","corrupt_size=","''","o","''","``",")",",","the","subject","and","the","object","(","``","corrupt_size=","''","s+o","''","``","or","``","corrupt_size=","''","s",",","o","''","``",")","(","default",":","`","``","s",",","o","''","`",")",".","ranking_strategy",":","str","Indicates","how","to","break","ties","when","a","test","triple","gets","the","same","rank","of","a","corruption",".","Can","be","one","of","the","three","types",":","`","``","best","''","`",",","`","``","middle","''","`",",","`","``","worst","''","`","(","default",":","`","``","worst","''","`",",","i.e.",",","the","worst","rank","is","assigned","to","the","test","triple",")",".","entities_subset",":","list","or","np.array","Subset","of","entities","to","be","used","for","generating","corruptions",".","callbacks",":","list","of","keras.callbacks.Callback","instances","List","of","callbacks","to","apply","during","evaluation",".","Returns","--","--","--","-","rank",":","np.array",",","shape","(","n",",","number","of","corrupted","sides",")","Ranking","of","test","triples","against","subject","corruptions","and\/or","object","corruptions",".","Example","--","--","--","-",">",">",">","from","ampligraph.datasets","import","load_fb15k_237",">",">",">","from","ampligraph.latent_features","import","ScoringBasedEmbeddingModel",">",">",">","from","ampligraph.evaluation.metrics","import","mrr_score",",","hits_at_n_score",",","mr_score",">",">",">","X","=","load_fb15k_237","(",")",">",">",">","model","=","ScoringBasedEmbeddingModel","(","eta=5",",",">",">",">","k=300",",",">",">",">","scoring_type='ComplEx","'",",",">",">",">","seed=0",")",">",">",">","model.compile","(","optimizer='adam","'",",","loss='nll","'",")",">",">",">","model.fit","(","X","[","'train","'","]",",",">",">",">","batch_size=10000",",",">",">",">","epochs=5",")","Epoch","1\/5","29\/29","[","==============================","]","-","2s","71ms\/step","-","loss",":","67361.3047","Epoch","2\/5","29\/29","[","==============================","]","-","1s","35ms\/step","-","loss",":","67318.6094","Epoch","3\/5","29\/29","[","==============================","]","-","1s","35ms\/step","-","loss",":","67020.0703","Epoch","4\/5","29\/29","[","==============================","]","-","1s","33ms\/step","-","loss",":","65867.3750","Epoch","5\/5","29\/29","[","==============================","]","-","1s","34ms\/step","-","loss",":","63517.9062",">",">",">","ranks","=","model.evaluate","(","X","[","'test","'","]",",",">",">",">","batch_size=100",",",">",">",">","corrupt_side=","'s",",","o","'",",",">",">",">","use_filter=","{","'train","'",":","X","[","'train","'","]",",",">",">",">","'valid","'",":","X","[","'valid","'","]",",",">",">",">","'test","'",":","X","[","'test","'","]",")",">",">",">","mr_score","(","ranks",")",",","mrr_score","(","ranks",")",",","hits_at_n_score","(","ranks",",","1",")",",","hits_at_n_score","(","ranks",",","10",")",",","len","(","ranks",")","28","triples","containing","invalid","keys","skipped","!","9","triples","containing","invalid","keys","skipped","!","2045\/2045","[","==============================","]","-","149s","73ms\/step","(","428.44671689989235",",","0.25761041025282316",",","0.1898179861043155",",","0.391965945787259",",","20438",")"]
code: def evaluate(
      self,
      x=None,
      batch_size=32,
      verbose=True,
      use_filter=False,
      corrupt_side="s,o",
      entities_subset=None,
      ranking_strategy="worst",
      callbacks=None,
      dataset_type="test",
  ):
      """
      Evaluate the inputs against corruptions and return ranks.
  
      Parameters
      ----------
      x: np.array, shape (n,3) or str or GraphDataLoader or AbstractGraphPartitioner
          Data OR Filename of the data file OR Data Handle to be used for training.
      batch_size: int
          Batch size to use during training.
          May be overridden if ``x`` is `GraphDataLoader` or `AbstractGraphPartitioner` instance
      verbose: bool
          Verbosity mode.
      use_filter: bool or dict
          Whether to use a filter of not. If a dictionary is specified, the data in the dict is concatenated
          and used as filter.
      corrupt_side: str
          Which side to corrupt of a triple to corrupt. It can be the subject (``corrupt_size="s"``),
          the object (``corrupt_size="o"``), the subject and the object (``corrupt_size="s+o"`` or
          ``corrupt_size="s,o"``) (default:`"s,o"`).
      ranking_strategy: str
          Indicates how to break ties when a test triple gets the same rank of a corruption.
          Can be one of the three types: `"best"`, `"middle"`, `"worst"` (default: `"worst"`, i.e.,
          the worst rank is assigned to the test triple).
      entities_subset: list or np.array
          Subset of entities to be used for generating corruptions.
      callbacks: list of keras.callbacks.Callback instances
          List of callbacks to apply during evaluation.
  
      Returns
      -------
      rank: np.array, shape (n, number of corrupted sides)
          Ranking of test triples against subject corruptions and/or object corruptions.
  
      Example
      -------
      >>> from ampligraph.datasets import load_fb15k_237
      >>> from ampligraph.latent_features import ScoringBasedEmbeddingModel
      >>> from ampligraph.evaluation.metrics import mrr_score, hits_at_n_score, mr_score
      >>> X = load_fb15k_237()
      >>> model = ScoringBasedEmbeddingModel(eta=5,
      >>>                                    k=300,
      >>>                                    scoring_type='ComplEx',
      >>>                                    seed=0)
      >>> model.compile(optimizer='adam', loss='nll')
      >>> model.fit(X['train'],
      >>>           batch_size=10000,
      >>>           epochs=5)
      Epoch 1/5
      29/29 [==============================] - 2s 71ms/step - loss: 67361.3047
      Epoch 2/5
      29/29 [==============================] - 1s 35ms/step - loss: 67318.6094
      Epoch 3/5
      29/29 [==============================] - 1s 35ms/step - loss: 67020.0703
      Epoch 4/5
      29/29 [==============================] - 1s 33ms/step - loss: 65867.3750
      Epoch 5/5
      29/29 [==============================] - 1s 34ms/step - loss: 63517.9062
      >>> ranks = model.evaluate(X['test'],
      >>>                        batch_size=100,
      >>>                        corrupt_side='s,o',
      >>>                        use_filter={'train': X['train'],
      >>>                                    'valid': X['valid'],
      >>>                                    'test': X['test'])
      >>> mr_score(ranks), mrr_score(ranks), hits_at_n_score(ranks, 1), hits_at_n_score(ranks, 10), len(ranks)
      28 triples containing invalid keys skipped!
      9 triples containing invalid keys skipped!
      2045/2045 [==============================] - 149s 73ms/step
      (428.44671689989235,
       0.25761041025282316,
       0.1898179861043155,
       0.391965945787259,
       20438)
      """
      # get the test set handler
      self.data_handler_test = data_adapter.DataHandler(
          x,
          batch_size=batch_size,
          dataset_type=dataset_type,
          epochs=1,
          use_filter=use_filter,
          use_indexer=self.data_indexer,
      )
  
      assert corrupt_side in [
          "s",
          "o",
          "s,o",
          "s+o",
      ], "Invalid value for corrupt_side"
      assert ranking_strategy in [
          "best",
          "middle",
          "worst",
      ], "Invalid value for ranking_strategy"
  
      self.corrupt_side = corrupt_side
      self.ranking_strategy = ranking_strategy
  
      self.entities_subset = tf.constant([])
      self.mapping_dict = tf.lookup.experimental.DenseHashTable(
          tf.int32, tf.int32, -1, -1, -2
      )
      if entities_subset is not None:
          entities_subset = self.data_indexer.get_indexes(
              entities_subset, "e"
          )
          self.entities_subset = tf.constant(
              entities_subset, dtype=tf.int32
          )
          self.mapping_dict.insert(
              self.entities_subset,
              tf.range(self.entities_subset.shape[0]),
          )
  
      # flag to indicate if we are using filter or not
      self.use_filter = (
          self.data_handler_test._parent_adapter.backend.use_filter
          or isinstance(
              self.data_handler_test._parent_adapter.backend.use_filter,
              dict,
          )
      )
  
      # Container that configures and calls `tf.keras.Callback`s.
      if not isinstance(callbacks, callbacks_module.CallbackList):
          callbacks = callbacks_module.CallbackList(
              callbacks,
              add_history=True,
              add_progbar=verbose != 0,
              model=self,
              verbose=verbose,
              epochs=1,
              steps=self.data_handler_test.inferred_steps,
          )
  
      test_function = self.make_test_function()
  
      # before test begins call this callback function
      callbacks.on_test_begin()
  
      self.all_ranks = []
  
      # enumerate over the data
      for _, iterator in self.data_handler_test.enumerate_epochs():
          # handle the stop iteration of data iterator in this scope
          with self.data_handler_test.catch_stop_iteration():
              # iterate over the dataset
              for step in self.data_handler_test.steps():
                  # before a batch is processed call this callback function
                  callbacks.on_test_batch_begin(step)
  
                  # process this batch
                  overall_rank = test_function(iterator)
                  # increment the rank by 1 (ranks returned are from (0 -
                  # n-1) so increment by 1
                  overall_rank += 1
                  # save the ranks of the batch triples
                  self.all_ranks.append(overall_rank)
                  # after a batch is processed call this callback function
                  callbacks.on_test_batch_end(step)
      # on test end call this method
      callbacks.on_test_end()
      # return ranks
      return np.concatenate(self.all_ranks)
  
imports: import tensorflow
  import copy
  import shelve
  import pickle
  import numpy
  import os
  import tempfile
  import logging
  from ampligraph.evaluation.metrics import mrr_score, hits_at_n_score, mr_score
  from ampligraph.datasets import data_adapter
  from ampligraph.datasets.partitioned_data_manager import PartitionDataManager
  from ampligraph.latent_features.layers.scoring.AbstractScoringLayer import SCORING_LAYER_REGISTRY
  from ampligraph.latent_features.layers.encoding import EmbeddingLookupLayer
  from ampligraph.latent_features.layers.calibration import CalibrationLayer
  from ampligraph.latent_features.layers.corruption_generation import CorruptionGenerationLayerTrain
  from ampligraph.datasets.data_indexer import DataIndexer
  from ampligraph.latent_features import optimizers
  from ampligraph.latent_features import loss_functions
  from ampligraph.evaluation import train_test_split_no_unseen
  from tensorflow.python.keras import callbacks
  from tensorflow.python.keras.engine import training_utils
  from tensorflow.python.eager import def_function
  from tensorflow.python.keras import metrics
  from tensorflow.python.keras.engine import compile_utils
label_desc: Use image node_id 30 for calling the ScoringBasedEmbeddingModel obj's underlying member method code with example usage: obj.evaluate(x, batch_size, verbose, use_filter, corrupt_side, entities_subset, ranking_strategy, callbacks, dataset_type) and returns: np