class_docstr: A collection of data which can be pre-processed in parallel on Spark.
class_docstr_tok: ["A","collection","of","data","which","can","be","pre-processed","in","parallel","on","Spark","."]
code: def deduplicates(self) -> Optional["SparkXShards"]:
      if self._get_class_name() == "pandas.core.frame.DataFrame":
          import pandas as pd
  
          df = self.to_spark_df()
          distinctDF = df.distinct()
          data_shards = spark_df_to_pd_sparkxshards(distinctDF)
          return data_shards
      else:
          invalidInputError(
              False,
              "Currently only support dedup() on XShards of Pandas DataFrame",
          )
      return None
  
imports: from py4j.protocol import Py4JError
  from bigdl.orca.data.utils import *
  from bigdl.orca import OrcaContext
  from bigdl.dllib.utils.common import get_node_and_core_number, get_spark_sql_context, get_spark_context
  from bigdl.dllib.utils import nest
  from bigdl.dllib.utils.log4Error import invalidInputError
  import numpy
  from typing import TYPE_CHECKING, Any
  from typing import Callable, Dict, List, Optional, Tuple, Union
label_desc: Use image node_id 18 for calling the SparkXShards obj's underlying member method code with example usage: obj.deduplicates() and returns: None, data_shards