class_docstr: A collection of data which can be pre-processed in parallel on Spark.
class_docstr_tok: ["A","collection","of","data","which","can","be","pre-processed","in","parallel","on","Spark","."]
docstr: Group the Shards with specified columns and then run aggregation. Optionally join the
  result with the original Shards.
  
  :param columns: str or a list of str. Columns to group the SparkXShards. If it is an
         empty list, aggregation is run directly without grouping. Default is [].
  :param agg: str, list or dict. Aggregate functions to be applied to grouped SparkXShards.
         Default is "count".
         Supported aggregate functions are: "max", "min", "count", "sum", "avg", "mean",
         "sumDistinct", "stddev", "stddev_pop", "variance", "var_pop", "skewness", "kurtosis",
         "collect_list", "collect_set", "approx_count_distinct", "first", "last".
         If agg is a str, then agg is the aggregate function and the aggregation is performed
         on all columns that are not in `columns`.
         If agg is a list of str, then agg is a list of aggregate function and the aggregation
         is performed on all columns that are not in `columns`.
         If agg is a single dict mapping from str to str, then the key is the column
         to perform aggregation on, and the value is the aggregate function.
         If agg is a single dict mapping from str to list, then the key is the
         column to perform aggregation on, and the value is list of aggregate functions.
  
         Examples:
         agg="sum"
         agg=["last", "stddev"]
         agg={"*":"count"}
         agg={"col_1":"sum", "col_2":["count", "mean"]}
  :param join: boolean. If True, join the aggregation result with original SparkXShards.
  
  :return: A new SparkXShards with aggregated column fields.
docstr_tok: ["Group","the","Shards","with","specified","columns","and","then","run","aggregation",".","Optionally","join","the","result","with","the","original","Shards",".",":","param","columns",":","str","or","a","list","of","str",".","Columns","to","group","the","SparkXShards",".","If","it","is","an","empty","list",",","aggregation","is","run","directly","without","grouping",".","Default","is","[","]",".",":","param","agg",":","str",",","list","or","dict",".","Aggregate","functions","to","be","applied","to","grouped","SparkXShards",".","Default","is","``","count","''",".","Supported","aggregate","functions","are",":","``","max","''",",","``","min","''",",","``","count","''",",","``","sum","''",",","``","avg","''",",","``","mean","''",",","``","sumDistinct","''",",","``","stddev","''",",","``","stddev_pop","''",",","``","variance","''",",","``","var_pop","''",",","``","skewness","''",",","``","kurtosis","''",",","``","collect_list","''",",","``","collect_set","''",",","``","approx_count_distinct","''",",","``","first","''",",","``","last","''",".","If","agg","is","a","str",",","then","agg","is","the","aggregate","function","and","the","aggregation","is","performed","on","all","columns","that","are","not","in","`","columns","`",".","If","agg","is","a","list","of","str",",","then","agg","is","a","list","of","aggregate","function","and","the","aggregation","is","performed","on","all","columns","that","are","not","in","`","columns","`",".","If","agg","is","a","single","dict","mapping","from","str","to","str",",","then","the","key","is","the","column","to","perform","aggregation","on",",","and","the","value","is","the","aggregate","function",".","If","agg","is","a","single","dict","mapping","from","str","to","list",",","then","the","key","is","the","column","to","perform","aggregation","on",",","and","the","value","is","list","of","aggregate","functions",".","Examples",":","agg=","''","sum","''","agg=","[","``","last","''",",","``","stddev","''","]","agg=","{","``","*","''",":","''","count","''","}","agg=","{","``","col_1","''",":","''","sum","''",",","``","col_2","''",":","[","``","count","''",",","``","mean","''","]","}",":","param","join",":","boolean",".","If","True",",","join","the","aggregation","result","with","original","SparkXShards",".",":","return",":","A","new","SparkXShards","with","aggregated","column","fields","."]
code: def group_by(
      self,
      columns: Union[str, List[str]] = [],
      agg: Union[
          Dict[str, List[str]], List[str], Dict[str, str], str
      ] = "count",
      join: bool = False,
  ) -> "SparkXShards":
      """
      Group the Shards with specified columns and then run aggregation. Optionally join the
      result with the original Shards.
  
      :param columns: str or a list of str. Columns to group the SparkXShards. If it is an
             empty list, aggregation is run directly without grouping. Default is [].
      :param agg: str, list or dict. Aggregate functions to be applied to grouped SparkXShards.
             Default is "count".
             Supported aggregate functions are: "max", "min", "count", "sum", "avg", "mean",
             "sumDistinct", "stddev", "stddev_pop", "variance", "var_pop", "skewness", "kurtosis",
             "collect_list", "collect_set", "approx_count_distinct", "first", "last".
             If agg is a str, then agg is the aggregate function and the aggregation is performed
             on all columns that are not in `columns`.
             If agg is a list of str, then agg is a list of aggregate function and the aggregation
             is performed on all columns that are not in `columns`.
             If agg is a single dict mapping from str to str, then the key is the column
             to perform aggregation on, and the value is the aggregate function.
             If agg is a single dict mapping from str to list, then the key is the
             column to perform aggregation on, and the value is list of aggregate functions.
  
             Examples:
             agg="sum"
             agg=["last", "stddev"]
             agg={"*":"count"}
             agg={"col_1":"sum", "col_2":["count", "mean"]}
      :param join: boolean. If True, join the aggregation result with original SparkXShards.
  
      :return: A new SparkXShards with aggregated column fields.
      """
  
      if self._get_class_name() != "pandas.core.frame.DataFrame":
          invalidInputError(
              False,
              "Currently only support sort() on XShards of Pandas DataFrame",
          )
  
      df = self.to_spark_df()
      sqlContext = get_spark_sql_context(get_spark_context())
      defaultPartitionNum = sqlContext.getConf(
          "spark.sql.shuffle.partitions"
      )
      partitionNum = df.rdd.getNumPartitions()
      sqlContext.setConf(
          "spark.sql.shuffle.partitions", str(partitionNum)
      )
  
      result_df = group_by_spark_df(df, columns, agg, join)
  
      agg_shards = spark_df_to_pd_sparkxshards(result_df)
      sqlContext.setConf(
          "spark.sql.shuffle.partitions", defaultPartitionNum
      )
      return agg_shards
  
imports: from py4j.protocol import Py4JError
  from bigdl.orca.data.utils import *
  from bigdl.orca import OrcaContext
  from bigdl.dllib.utils.common import get_node_and_core_number, get_spark_sql_context, get_spark_context
  from bigdl.dllib.utils import nest
  from bigdl.dllib.utils.log4Error import invalidInputError
  import numpy
  from typing import TYPE_CHECKING, Any
  from typing import Callable, Dict, List, Optional, Tuple, Union
label_desc: Use image node_id 26 for calling the SparkXShards obj's underlying member method code with example usage: obj.group_by(columns, agg, join) and returns: agg_shards