docstr: Clean DataSet from full duplicates.
docstr_tok: ["Clean","DataSet","from","full","duplicates","."]
code: def __clean_duplicates(self, silent_mode: bool = False):
      """Clean DataSet from full duplicates."""
      # self.logger.info("Clean full duplicates")
      nrows = len(self.data)
      if nrows == 0:
          return
      # Remove absolute duplicates (exclude system_record_id)
      unique_columns = self.data.columns.tolist()
      unique_columns.remove(SYSTEM_RECORD_ID)
      self.logger.info(
          f"Dataset shape before clean duplicates: {self.data.shape}"
      )
      self.data.drop_duplicates(subset=unique_columns, inplace=True)
      self.logger.info(
          f"Dataset shape after clean duplicates: {self.data.shape}"
      )
      nrows_after_full_dedup = len(self.data)
      share_full_dedup = 100 * (1 - nrows_after_full_dedup / nrows)
      if share_full_dedup > 0:
          msg = bundle.get("dataset_full_duplicates").format(
              share_full_dedup
          )
          self.logger.warning(msg)
          # if not silent_mode:
          #     print(msg)
          # self.warning_counter.increment()
      target_column = self.etalon_def_checked.get(
          FileColumnMeaningType.TARGET.value
      )
      if target_column is not None:
          unique_columns.remove(target_column)
          marked_duplicates = self.data.duplicated(
              subset=unique_columns, keep=False
          )
          if marked_duplicates.sum() > 0:
              dups_indices = self.data[
                  marked_duplicates
              ].index.to_list()
              nrows_after_tgt_dedup = len(
                  self.data.drop_duplicates(subset=unique_columns)
              )
              num_dup_rows = (
                  nrows_after_full_dedup - nrows_after_tgt_dedup
              )
              share_tgt_dedup = (
                  100 * num_dup_rows / nrows_after_full_dedup
              )
  
              msg = bundle.get("dataset_diff_target_duplicates").format(
                  share_tgt_dedup, num_dup_rows, dups_indices
              )
              self.logger.warning(msg)
              if not silent_mode:
                  print(msg)
              self.data.drop_duplicates(
                  subset=unique_columns, keep=False, inplace=True
              )
              self.logger.info(
                  f"Dataset shape after clean invalid target duplicates: {self.data.shape}"
              )
  
imports: import csv
  import hashlib
  import logging
  import tempfile
  import time
  from ipaddress import IPv4Address, IPv6Address, _BaseAddress, ip_address
  from pathlib import Path
  from typing import Any, Callable, Dict, List, Optional, Tuple, Union
  import numpy
  import pandas
  from pandas.api.types import is_bool_dtype
  from pandas.api.types import is_datetime64_any_dtype
  from pandas.api.types import is_float_dtype, is_integer_dtype, is_numeric_dtype, is_string_dtype
  from pandas.core.dtypes.common import is_period_dtype
  from upgini.errors import ValidationError
  from upgini.http import ProgressStage, SearchProgress, get_rest_client
  from upgini.metadata import EVAL_SET_INDEX, SYSTEM_COLUMNS, SYSTEM_RECORD_ID, TARGET, DataType, FeaturesFilter, FileColumnMeaningType, FileColumnMetadata, FileMetadata, FileMetrics, ModelTaskType, NumericInterval, RuntimeParameters, SearchCustomization
  from upgini.normalizer.phone_normalizer import PhoneNormalizer
  from upgini.resource_bundle import bundle
  from upgini.sampler.random_under_sampler import RandomUnderSampler
  from upgini.search_task import SearchTask
  from upgini.utils import combine_search_keys
  from upgini.utils.email_utils import EmailSearchKeyConverter
  from upgini.utils.warning_counter import WarningCounter
label_desc: Use image node_id 11 for calling the Dataset obj's underlying member method code with example usage: obj.__clean_duplicates(silent_mode) without return types