docstr: Validate DataSet
docstr_tok: ["Validate","DataSet"]
code: def __validate_dataset(
      self, validate_target: bool, silent_mode: bool
  ):
      """Validate DataSet"""
      # self.logger.info("validating etalon")
      target = self.etalon_def_checked.get(
          FileColumnMeaningType.TARGET.value
      )
      if validate_target:
          if target is None:
              raise ValidationError(
                  bundle.get("dataset_missing_target")
              )
  
          target_value = self.__target_value()
          target_items = target_value.nunique()
          if target_items == 1:
              raise ValidationError(
                  bundle.get("dataset_constant_target")
              )
          elif target_items == 0:
              raise ValidationError(bundle.get("dataset_empty_target"))
  
          # if self.task_type != ModelTaskType.MULTICLASS:
          #     self.data[target] = self.data[target].apply(pd.to_numeric, errors="coerce")
  
      keys_to_validate = [
          key
          for search_group in self.search_keys_checked
          for key in search_group
          if self.columns_renaming.get(key)
          != EmailSearchKeyConverter.EMAIL_ONE_DOMAIN_COLUMN_NAME
      ]
      mandatory_columns = [target]
      columns_to_validate = mandatory_columns.copy()
      columns_to_validate.extend(keys_to_validate)
      columns_to_validate = set(
          [i for i in columns_to_validate if i is not None]
      )
  
      nrows = len(self.data)
      validation_stats = {}
      self.data["valid_keys"] = 0
      self.data["valid_mandatory"] = True
  
      all_valid_status = bundle.get("validation_all_valid_status")
      some_invalid_status = bundle.get("validation_some_invalid_status")
      all_invalid_status = bundle.get("validation_all_invalid_status")
      all_valid_message = bundle.get("validation_all_valid_message")
      invalid_message = bundle.get("validation_invalid_message")
  
      for col in columns_to_validate:
          self.data[f"{col}_is_valid"] = ~self.data[col].isnull()
          if validate_target and target is not None and col == target:
              self.data.loc[
                  self.data[target] == np.Inf, f"{col}_is_valid"
              ] = False
  
          if col in mandatory_columns:
              self.data["valid_mandatory"] = (
                  self.data["valid_mandatory"]
                  & self.data[f"{col}_is_valid"]
              )
  
          invalid_values = list(self.data.loc[self.data[f"{col}_is_valid"] == 0, col].head().values)  # type: ignore
          valid_share = self.data[f"{col}_is_valid"].sum() / nrows
          original_col_name = self.columns_renaming[col]
          validation_stats[original_col_name] = {}
          if valid_share == 1:
              valid_status = all_valid_status
              valid_message = all_valid_message
          elif 0 < valid_share < 1:
              valid_status = some_invalid_status
              valid_message = invalid_message.format(
                  100 * (1 - valid_share), invalid_values
              )
          else:
              valid_status = all_invalid_status
              valid_message = invalid_message.format(
                  100 * (1 - valid_share), invalid_values
              )
          validation_stats[original_col_name][
              "valid_status"
          ] = valid_status
          validation_stats[original_col_name][
              "valid_message"
          ] = valid_message
  
          if col in keys_to_validate:
              self.data["valid_keys"] = (
                  self.data["valid_keys"] + self.data[f"{col}_is_valid"]
              )
          self.data.drop(columns=f"{col}_is_valid", inplace=True)
  
      self.data["is_valid"] = self.data["valid_keys"] > 0
      self.data["is_valid"] = (
          self.data["is_valid"] & self.data["valid_mandatory"]
      )
      self.data.drop(
          columns=["valid_keys", "valid_mandatory"], inplace=True
      )
  
      drop_idx = self.data[self.data["is_valid"] != 1].index  # type: ignore
      self.data.drop(index=drop_idx, inplace=True)  # type: ignore
      self.data.drop(columns=["is_valid"], inplace=True)
  
      if not silent_mode:
          df_stats = pd.DataFrame.from_dict(
              validation_stats, orient="index"
          )
          df_stats.reset_index(inplace=True)
          name_header = bundle.get("validation_column_name_header")
          status_header = bundle.get("validation_status_header")
          description_header = bundle.get("validation_descr_header")
          df_stats.columns = [
              name_header,
              status_header,
              description_header,
          ]
          try:
              import html
  
              from IPython.display import HTML, display  # type: ignore
  
              _ = get_ipython()  # type: ignore
  
              text_color = bundle.get("validation_text_color")
              colormap = {
                  all_valid_status: bundle.get(
                      "validation_all_valid_color"
                  ),
                  some_invalid_status: bundle.get(
                      "validation_some_invalid_color"
                  ),
                  all_invalid_status: bundle.get(
                      "validation_all_invalid_color"
                  ),
              }
  
              def map_color(text) -> str:
                  return (
                      f"<td style='background-color:{colormap[text]};color:{text_color}'>{text}</td>"
                      if text in colormap
                      else f"<td>{text}</td>"
                  )
  
              df_stats[description_header] = df_stats[
                  description_header
              ].apply(lambda x: html.escape(x))
              html_stats = (
                  "<table>"
                  + "<tr>"
                  + "".join(
                      f"<th style='font-weight:bold'>{column}</th>"
                      for column in df_stats.columns
                  )
                  + "</tr>"
                  + "".join(
                      "<tr>"
                      + "".join(map(map_color, row[1:]))
                      + "</tr>"
                      for row in df_stats.itertuples()
                  )
                  + "</table>"
              )
              display(HTML(html_stats))
          except (ImportError, NameError):
              print(df_stats)
  
      if len(self.data) == 0:
          raise ValidationError(bundle.get("all_search_keys_invalid"))
  
imports: import csv
  import hashlib
  import logging
  import tempfile
  import time
  from ipaddress import IPv4Address, IPv6Address, _BaseAddress, ip_address
  from pathlib import Path
  from typing import Any, Callable, Dict, List, Optional, Tuple, Union
  import numpy
  import pandas
  from pandas.api.types import is_bool_dtype
  from pandas.api.types import is_datetime64_any_dtype
  from pandas.api.types import is_float_dtype, is_integer_dtype, is_numeric_dtype, is_string_dtype
  from pandas.core.dtypes.common import is_period_dtype
  from upgini.errors import ValidationError
  from upgini.http import ProgressStage, SearchProgress, get_rest_client
  from upgini.metadata import EVAL_SET_INDEX, SYSTEM_COLUMNS, SYSTEM_RECORD_ID, TARGET, DataType, FeaturesFilter, FileColumnMeaningType, FileColumnMetadata, FileMetadata, FileMetrics, ModelTaskType, NumericInterval, RuntimeParameters, SearchCustomization
  from upgini.normalizer.phone_normalizer import PhoneNormalizer
  from upgini.resource_bundle import bundle
  from upgini.sampler.random_under_sampler import RandomUnderSampler
  from upgini.search_task import SearchTask
  from upgini.utils import combine_search_keys
  from upgini.utils.email_utils import EmailSearchKeyConverter
  from upgini.utils.warning_counter import WarningCounter
label_desc: Use image node_id 34 for calling the Dataset obj's underlying member method code with example usage: obj.__validate_dataset(validate_target, silent_mode) and returns: unknown