class_docstr: A collection of data which can be pre-processed in parallel.
class_docstr_tok: ["A","collection","of","data","which","can","be","pre-processed","in","parallel","."]
docstr: Partition local in memory data and form a SparkXShards
  
  :param data: np.ndarray, a tuple, list, dict of np.ndarray, or a nested structure
  made of tuple, list, dict with ndarray as the leaf value
  :param num_shards: the number of shards that the data will be partitioned into
  :return: a SparkXShards
docstr_tok: ["Partition","local","in","memory","data","and","form","a","SparkXShards",":","param","data",":","np.ndarray",",","a","tuple",",","list",",","dict","of","np.ndarray",",","or","a","nested","structure","made","of","tuple",",","list",",","dict","with","ndarray","as","the","leaf","value",":","param","num_shards",":","the","number","of","shards","that","the","data","will","be","partitioned","into",":","return",":","a","SparkXShards"]
code: def partition(
      data: Union[
          "ndarray",
          List["ndarray"],
          Tuple["ndarray", "ndarray"],
          Dict[
              str, Union["ndarray", Tuple["ndarray"], List["ndarray"]]
          ],
      ],
      num_shards: Optional[int] = None,
  ) -> "SparkXShards":
      """
  
      Partition local in memory data and form a SparkXShards
  
      :param data: np.ndarray, a tuple, list, dict of np.ndarray, or a nested structure
      made of tuple, list, dict with ndarray as the leaf value
      :param num_shards: the number of shards that the data will be partitioned into
      :return: a SparkXShards
      """
      sc = OrcaContext.get_spark_context()
      node_num, core_num = get_node_and_core_number()
      shard_num = (
          node_num * core_num if num_shards is None else num_shards
      )
      import numpy as np
  
      type_err_msg = """
  The types supported in bigdl.orca.data.XShards.partition are
  1. np.ndarray
  2. a tuple, list, dict of np.ndarray
  3. nested structure made of tuple, list, dict with ndarray as the leaf value
  
  But got data of type {}
          """.format(
          type(data)
      )
      supported_types = {list, tuple, dict}
      if isinstance(data, np.ndarray):
          if data.shape[0] < shard_num:
              invalidInputError(
                  False,
                  "The length of data {} is smaller than the total number "
                  "of shards {}. Please adjust the num_shards option to be "
                  "at most {}.".format(
                      data.shape[0], shard_num, data.shape[0]
                  ),
              )
          arrays = np.array_split(data, shard_num)
          rdd = sc.parallelize(arrays)
      else:
          invalidInputError(type(data) in supported_types, type_err_msg)
          flattened = nest.flatten(data)
          data_length = len(flattened[0])
          data_to_be_shard = []  # type:ignore
          if data_length < shard_num:
              invalidInputError(
                  False,
                  "The length of data {} is smaller than the total number "
                  "of shards {}. Please adjust the num_shards option to be "
                  "at most {}.".format(
                      data_length, shard_num, data_length
                  ),
              )
          for i in range(shard_num):
              data_to_be_shard.append([])
          for x in flattened:
              invalidInputError(
                  len(x) == data_length,
                  "the ndarrays in data must all have the same size in first"
                  " dimension, got first ndarray of size {} and"
                  " another {}".format(data_length, len(x)),
              )
              x_parts = np.array_split(x, shard_num)
              for idx, x_part in enumerate(x_parts):
                  data_to_be_shard[idx].append(x_part)
  
          data_to_be_shard = [
              nest.pack_sequence_as(data, shard)
              for shard in data_to_be_shard
          ]
          rdd = sc.parallelize(data_to_be_shard)
  
      data_shards = SparkXShards(rdd)
      return data_shards
  
imports: from py4j.protocol import Py4JError
  from bigdl.orca.data.utils import *
  from bigdl.orca import OrcaContext
  from bigdl.dllib.utils.common import get_node_and_core_number, get_spark_sql_context, get_spark_context
  from bigdl.dllib.utils import nest
  from bigdl.dllib.utils.log4Error import invalidInputError
  import numpy
  from typing import TYPE_CHECKING, Any
  from typing import Callable, Dict, List, Optional, Tuple, Union
label_desc: Use image node_id 5 for calling the XShards obj's underlying member method code with example usage: obj.partition(data, num_shards) and returns: data_shards