class_docstr: Class for linking of entity substrings in the document to entities in Wikidata
class_docstr_tok: ["Class","for","linking","of","entity","substrings","in","the","document","to","entities","in","Wikidata"]
code: def link_entities(
      self,
      substr_list: List[str],
      offsets_list: List[List[int]],
      tags_list: List[str],
      probas_list: List[float],
      sentences_list: List[str],
      sentences_offsets_list: List[List[int]],
      entities_to_link: List[int],
  ) -> Tuple[
      List[Any],
      List[Any],
      List[List[Union[str, Any]]],
      List[List[Union[str, Any]]],
  ]:
      log.debug(
          f"substr_list {substr_list} tags_list {tags_list} probas {probas_list} offsets_list {offsets_list}"
      )
      ids_list, conf_list, pages_list, label_list, descr_list = (
          [],
          [],
          [],
          [],
          [],
      )
      if substr_list:
          entities_scores_list = []
          cand_ent_scores_list = []
          for substr, tags, proba in zip(
              substr_list, tags_list, probas_list
          ):
              for old_symb, new_symb in [
                  ("'s", ""),
                  ("@", ""),
                  ("  ", " "),
                  (".", ""),
                  (",", ""),
                  ("-", " "),
                  ("'", " "),
                  ("!", ""),
                  (":", ""),
                  ("&", ""),
                  ("/", " "),
                  ('"', ""),
                  ("  ", " "),
              ]:
                  substr = substr.replace(old_symb, new_symb)
              substr = substr.strip()
              cand_ent_init = defaultdict(set)
              if len(substr) > 1:
                  if isinstance(tags, str):
                      tags = [tags]
                  tags = [tag.lower() for tag in tags]
                  if tags and not isinstance(tags[0], (list, tuple)):
                      tags = [(tag, 1.0) for tag in tags]
                  if tags and tags[0][0] == "e":
                      use_tags_flag = False
                  else:
                      use_tags_flag = True
                  cand_ent_init = self.find_exact_match(
                      substr, tags, use_tags=use_tags_flag
                  )
                  new_substr = re.sub(
                      r"\b([a-z]{1}) ([a-z]{1})\b", r"\1\2", substr
                  )
                  if substr != new_substr:
                      new_cand_ent_init = self.find_exact_match(
                          new_substr, tags, use_tags=use_tags_flag
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, new_cand_ent_init
                      )
  
                  init_substr_split = substr.lower().split(" ")
                  if tags[0][0] in {"person", "work_of_art"}:
                      substr_split = [
                          word
                          for word in substr.lower().split(" ")
                          if len(word) > 0
                      ]
                  else:
                      substr_split = [
                          word
                          for word in substr.lower().split(" ")
                          if word not in self.stopwords
                          and len(word) > 0
                      ]
  
                  substr_split_lemm = [
                      self.nlp(tok)[0].lemma_ for tok in substr_split
                  ]
                  substr_lemm = " ".join(substr_split_lemm)
                  if substr_split != substr_split_lemm or (
                      tags[0][0] == "work_of_art"
                      and len(substr_split) != len(init_substr_split)
                  ):
                      new_cand_ent_init = self.find_fuzzy_match(
                          substr_split, tags, use_tags=use_tags_flag
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, new_cand_ent_init
                      )
                  if substr_split != substr_split_lemm:
                      new_cand_ent_init = self.find_exact_match(
                          substr_lemm, tags, use_tags=use_tags_flag
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, new_cand_ent_init
                      )
                      new_cand_ent_init = self.find_fuzzy_match(
                          substr_split_lemm,
                          tags,
                          use_tags=use_tags_flag,
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, new_cand_ent_init
                      )
  
                  all_low_conf = self.define_all_low_conf(
                      cand_ent_init, 1.0
                  )
                  (
                      clean_tags,
                      corr_tags,
                      corr_clean_tags,
                  ) = self.correct_tags(tags)
                  log.debug(
                      f"substr: {substr} --- lemm: {substr_split_lemm} --- tags: {tags} --- corr_tags: "
                      f"{corr_tags} --- all_low_conf: {all_low_conf} --- cand_ent_init: {len(cand_ent_init)}"
                  )
  
                  if (not cand_ent_init or all_low_conf) and corr_tags:
                      corr_cand_ent_init = self.find_exact_match(
                          substr, corr_tags, use_tags=use_tags_flag
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, corr_cand_ent_init
                      )
                      if substr_split != substr_split_lemm:
                          new_cand_ent_init = self.find_exact_match(
                              substr_lemm,
                              corr_tags,
                              use_tags=use_tags_flag,
                          )
                          cand_ent_init = self.unite_dicts(
                              cand_ent_init, new_cand_ent_init
                          )
                          new_cand_ent_init = self.find_fuzzy_match(
                              substr_split_lemm,
                              corr_tags,
                              use_tags=use_tags_flag,
                          )
                          cand_ent_init = self.unite_dicts(
                              cand_ent_init, new_cand_ent_init
                          )
  
                  if (
                      not cand_ent_init
                      and len(substr_split) == 1
                      and self.word_searcher
                  ):
                      corr_words = self.word_searcher(
                          substr_split[0],
                          set(clean_tags + corr_clean_tags),
                      )
                      if corr_words:
                          cand_ent_init = self.find_exact_match(
                              corr_words[0],
                              tags + corr_tags,
                              use_tags=use_tags_flag,
                          )
  
                  if not cand_ent_init and len(substr_split) > 1:
                      cand_ent_init = self.find_fuzzy_match(
                          substr_split, tags
                      )
  
                  all_low_conf = self.define_all_low_conf(
                      cand_ent_init, 0.85
                  )
                  if (not cand_ent_init or all_low_conf) and tags[0][
                      0
                  ] != "t":
                      use_tags_flag = False
                      new_cand_ent_init = self.find_exact_match(
                          substr, tags, use_tags=use_tags_flag
                      )
                      cand_ent_init = self.unite_dicts(
                          cand_ent_init, new_cand_ent_init
                      )
                      if substr_split != substr_split_lemm and (
                          tags[0][0] == "e" or not cand_ent_init
                      ):
                          new_cand_ent_init = self.find_fuzzy_match(
                              substr_split, tags, use_tags=use_tags_flag
                          )
                          cand_ent_init = self.unite_dicts(
                              cand_ent_init, new_cand_ent_init
                          )
                          new_cand_ent_init = self.find_fuzzy_match(
                              substr_split_lemm,
                              tags,
                              use_tags=use_tags_flag,
                          )
                          cand_ent_init = self.unite_dicts(
                              cand_ent_init, new_cand_ent_init
                          )
  
              cand_ent_scores = []
              for entity in cand_ent_init:
                  entities_scores = list(cand_ent_init[entity])
                  entities_scores = sorted(
                      entities_scores,
                      key=lambda x: (x[0], x[2], x[1]),
                      reverse=True,
                  )
                  cand_ent_scores.append(
                      ([entity] + list(entities_scores[0]))
                  )
  
              cand_ent_scores = sorted(
                  cand_ent_scores,
                  key=lambda x: (x[1], x[3], x[2]),
                  reverse=True,
              )
              cand_ent_scores = cand_ent_scores[
                  : self.num_entities_for_bert_ranking
              ]
              cand_ent_scores_list.append(cand_ent_scores)
              entity_ids = [elem[0] for elem in cand_ent_scores]
              scores = [elem[1:4] for elem in cand_ent_scores]
              conf_list.append(scores)
              entities_scores_list.append(
                  {
                      entity_id: entity_scores
                      for entity_id, entity_scores in zip(
                          entity_ids, scores
                      )
                  }
              )
              ids_list.append(entity_ids)
              pages = [elem[4] for elem in cand_ent_scores]
              entity_labels = [elem[5] for elem in cand_ent_scores]
              pages_list.append(
                  {
                      entity_id: page
                      for entity_id, page in zip(entity_ids, pages)
                  }
              )
              label_list.append(
                  {
                      entity_id: entity_label
                      for entity_id, entity_label in zip(
                          entity_ids, entity_labels
                      )
                  }
              )
              descr_list.append([elem[6] for elem in cand_ent_scores])
  
          scores_dict = {}
          if self.use_connections and self.kb:
              scores_dict = self.rank_by_connections(ids_list)
  
          substr_lens = [
              len(entity_substr.split())
              for entity_substr in substr_list
          ]
          ids_list, conf_list = self.rank_by_description(
              substr_list,
              tags_list,
              offsets_list,
              ids_list,
              descr_list,
              entities_scores_list,
              sentences_list,
              sentences_offsets_list,
              substr_lens,
              scores_dict,
          )
      label_list = [
          [label_dict.get(entity_id, "") for entity_id in entity_ids]
          for entity_ids, label_dict in zip(ids_list, label_list)
      ]
      pages_list = [
          [pages_dict.get(entity_id, "") for entity_id in entity_ids]
          for entity_ids, pages_dict in zip(ids_list, pages_list)
      ]
  
      f_ids_list, f_conf_list, f_pages_list, f_label_list = (
          [],
          [],
          [],
          [],
      )
      for ids, confs, pages, labels, add_flag in zip(
          ids_list, conf_list, pages_list, label_list, entities_to_link
      ):
          if add_flag:
              f_ids_list.append(ids)
              f_conf_list.append(confs)
              f_pages_list.append(pages)
              f_label_list.append(labels)
      return f_ids_list, f_conf_list, f_pages_list, f_label_list
  
imports: import re
  import sqlite3
  from logging import getLogger
  from typing import List, Dict, Tuple, Any, Union
  from collections import defaultdict
  import nltk
  import spacy
  from hdt import HDTDocument
  from nltk.corpus import stopwords
  from rapidfuzz import fuzz
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
  from deeppavlov.core.models.serializable import Serializable
  from deeppavlov.models.entity_extraction.find_word import WordSearcher
label_desc: Use image node_id 5 for calling the EntityLinker obj's underlying member method code with example usage: obj.link_entities(substr_list, offsets_list, tags_list, probas_list, sentences_list, sentences_offsets_list, entities_to_link) and returns: f_ids_list,  f_conf_list,  f_pages_list,  f_label_list