class_docstr: Сlass for storing, cleaning and processing your dataset
class_docstr_tok: ["\u0421lass","for","storing",",","cleaning","and","processing","your","dataset"]
docstr: Description of preproc_data:
      dataset preprocessing function
  
  Args:
      X_train=None (pd.DataFrame):
      X_test=None (pd.DataFrame):
      y_train=None (pd.DataFrame):
      cat_features=None (list):
      cat_encoder_names=None (list):
      clean_nan=True (Bool):
      num_generator_features=True (Bool):
      group_generator_features=True (Bool):
  
  Returns:
      X_train (pd.DataFrame)
      X_test (pd.DataFrame)
docstr_tok: ["Description","of","preproc_data",":","dataset","preprocessing","function","Args",":","X_train=None","(","pd.DataFrame",")",":","X_test=None","(","pd.DataFrame",")",":","y_train=None","(","pd.DataFrame",")",":","cat_features=None","(","list",")",":","cat_encoder_names=None","(","list",")",":","clean_nan=True","(","Bool",")",":","num_generator_features=True","(","Bool",")",":","group_generator_features=True","(","Bool",")",":","Returns",":","X_train","(","pd.DataFrame",")","X_test","(","pd.DataFrame",")"]
code: def preproc_data(
      self,
      X_train=None,
      X_test=None,
      y_train=None,
      cat_features=None,
      cat_encoder_names=None,
      clean_nan=True,
      num_generator_features=True,
      group_generator_features=True,
      target_enc_cat_features=True,
      normalization=True,
      verbose=1,
  ):
      """
      Description of preproc_data:
          dataset preprocessing function
  
      Args:
          X_train=None (pd.DataFrame):
          X_test=None (pd.DataFrame):
          y_train=None (pd.DataFrame):
          cat_features=None (list):
          cat_encoder_names=None (list):
          clean_nan=True (Bool):
          num_generator_features=True (Bool):
          group_generator_features=True (Bool):
  
      Returns:
          X_train (pd.DataFrame)
          X_test (pd.DataFrame)
  
      """
  
      #### Sometimes there are duplicates in column names. You must remove them here. ###
      cat_features = find_remove_duplicates(cat_features)
  
      # concat datasets for correct processing.
      df_train = X_train.copy()
  
      if X_test is None:
          data = df_train
          test_data = None  ### Set test_data to None if X_test is None
      else:
          test_data = X_test.copy()
          test_data = remove_duplicate_cols_in_dataset(test_data)
          data = copy.deepcopy(df_train)
  
      data = remove_duplicate_cols_in_dataset(data)
  
      # object & num features
      object_features = list(
          data.columns[
              (data.dtypes == "object") | (data.dtypes == "category")
          ]
      )
      num_features = list(
          set(data.columns)
          - set(cat_features)
          - set(object_features)
          - {"test"}
      )
      encodet_features_names = list(
          set(object_features + list(cat_features))
      )
  
      original_number_features = len(encodet_features_names)
      count_number_features = df_train.shape[1]
  
      self.encodet_features_names = encodet_features_names
      self.num_features_names = num_features
      self.binary_features_names = []
  
      # LabelEncode all Binary Features - leave the rest alone
      cols = data.columns.tolist()
      #### This sometimes errors because there are duplicate columns in a dataset ###
      print("LabelEncode all Boolean Features. Leave the rest alone")
      for feature in cols:
          if data[feature].dtype == bool:
              print("    boolean feature = ", feature)
  
      for feature in cols:
          if data[feature].dtype == bool:
              data[feature] = data[feature].astype("category").cat.codes
              if test_data is not None:
                  test_data[feature] = (
                      test_data[feature].astype("category").cat.codes
                  )
              self.binary_features_names.append(feature)
  
      # Convert all Category features "Category" type variables if no encoding is specified
      cat_only_encoders = [
          x
          for x in self.cat_encoder_names
          if x in self.cat_encoders_names
      ]
      if len(cat_only_encoders) > 0:
          ### Just skip if this encoder is not in the list of category encoders ##
          if encodet_features_names:
              if cat_encoder_names is None:
                  for feature in encodet_features_names:
                      data[feature] = data[feature].fillna("missing")
                      data[feature] = (
                          data[feature].astype("category").cat.codes
                      )
                      if test_data is not None:
                          test_data[feature] = test_data[
                              feature
                          ].fillna("missing")
                          test_data[feature] = (
                              test_data[feature]
                              .astype("category")
                              .cat.codes
                          )
              else:
                  #### If an encoder is specified, then use that encoder to transform categorical variables
                  if verbose > 0:
                      print("> Generate Categorical Encoded features")
  
                  copy_cat_encoder_names = copy.deepcopy(
                      cat_encoder_names
                  )
                  for encoder_name in copy_cat_encoder_names:
                      if verbose > 0:
                          print(
                              " + To know more, click: %s"
                              % self.cat_encoders_names[encoder_name][1]
                          )
                      (
                          data_encodet,
                          train_encoder,
                      ) = self.gen_cat_encodet_features(
                          data[encodet_features_names], encoder_name
                      )
                      if not isinstance(data_encodet, str):
                          data = pd.concat([data, data_encodet], axis=1)
                      if test_data is not None:
                          (
                              test_encodet,
                              _,
                          ) = self.gen_cat_encodet_features(
                              test_data[encodet_features_names],
                              train_encoder,
                          )
                          if not isinstance(test_encodet, str):
                              test_data = pd.concat(
                                  [test_data, test_encodet], axis=1
                              )
  
                      if verbose > 0:
                          if not isinstance(data_encodet, str):
                              addl_features = (
                                  data_encodet.shape[1]
                                  - original_number_features
                              )
                              count_number_features += addl_features
                              print(
                                  " + added ",
                                  addl_features,
                                  " additional Features using",
                                  encoder_name,
                              )
  
      # Generate Target related Encoder features for cat variables:
  
      target_encoders = [
          x
          for x in self.cat_encoder_names
          if x in self.target_encoders_names_list
      ]
      if len(target_encoders) > 0:
          target_enc_cat_features = True
      if target_enc_cat_features:
          if encodet_features_names:
              if verbose > 0:
                  print(
                      "> Generate Target Encoded categorical features"
                  )
  
              if len(target_encoders) == 0:
                  target_encoders = [
                      "TargetEncoder"
                  ]  ### set the default as TargetEncoder if nothing is specified
              copy_target_encoders = copy.deepcopy(target_encoders)
              for encoder_name in copy_target_encoders:
                  if verbose > 0:
                      print(
                          " + To know more, click: %s"
                          % self.target_encoders_names[encoder_name][1]
                      )
                  (
                      data_encodet,
                      train_encoder,
                  ) = self.gen_target_encodet_features(
                      data[encodet_features_names],
                      self.y_train_source,
                      encoder_name,
                  )
                  if not isinstance(data_encodet, str):
                      data = pd.concat([data, data_encodet], axis=1)
  
                  if test_data is not None:
                      (
                          test_encodet,
                          _,
                      ) = self.gen_target_encodet_features(
                          test_data[encodet_features_names],
                          "",
                          train_encoder,
                      )
                      if not isinstance(test_encodet, str):
                          test_data = pd.concat(
                              [test_data, test_encodet], axis=1
                          )
  
              if verbose > 0:
                  if not isinstance(data_encodet, str):
                      addl_features = (
                          data_encodet.shape[1]
                          - original_number_features
                      )
                      count_number_features += addl_features
                      print(
                          " + added ",
                          len(encodet_features_names),
                          " additional Features using ",
                          encoder_name,
                      )
  
      # Clean NaNs in Numeric variables only
      if clean_nan:
          if verbose > 0:
              print("> Cleaned NaNs in numeric features")
          data = self.clean_nans(data, cols=num_features)
          if test_data is not None:
              test_data = self.clean_nans(test_data, cols=num_features)
          ### Sometimes, train has nulls while test doesn't and vice versa
          if test_data is not None:
              rem_cols = left_subtract(list(data), list(test_data))
              if len(rem_cols) > 0:
                  for rem_col in rem_cols:
                      test_data[rem_col] = 0
              elif len(left_subtract(list(test_data), list(data))) > 0:
                  rem_cols = left_subtract(list(test_data), list(data))
                  for rem_col in rem_cols:
                      data[rem_col] = 0
              else:
                  print(" + test and train have similar NaN columns")
  
      # Generate interaction features for Numeric variables
      if num_generator_features:
          if len(num_features) > 1:
              if verbose > 0:
                  print(
                      "> Generate Interactions features among Numeric variables"
                  )
              fe_df = self.gen_numeric_interaction_features(
                  data[num_features],
                  num_features,
                  operations=["/", "*", "-", "+"],
              )
  
              if not isinstance(fe_df, str):
                  data = pd.concat([data, fe_df], axis=1)
              if test_data is not None:
                  fe_test = self.gen_numeric_interaction_features(
                      test_data[num_features],
                      num_features,
                      operations=["/", "*", "-", "+"],
                  )
                  if not isinstance(fe_test, str):
                      test_data = pd.concat(
                          [test_data, fe_test], axis=1
                      )
  
              if verbose > 0:
                  if not isinstance(fe_df, str):
                      addl_features = fe_df.shape[1]
                      count_number_features += addl_features
                      print(
                          " + added ",
                          addl_features,
                          " Interaction Features ",
                      )
  
      # Generate Group Encoded Features for Numeric variables only using all Categorical variables
      if group_generator_features:
          if encodet_features_names and num_features:
              if verbose > 0:
                  print("> Generate Group-by Encoded Features")
                  print(
                      " + To know more, click: %s"
                      % self.target_encoders_names["JamesSteinEncoder"][
                          1
                      ]
                  )
  
              for num_col in num_features:
                  (
                      data_encodet,
                      train_group_encoder,
                  ) = self.gen_groupby_cat_encode_features(
                      data,
                      encodet_features_names,
                      num_col,
                  )
                  if not isinstance(data_encodet, str):
                      data = pd.concat([data, data_encodet], axis=1)
                  if test_data is not None:
                      (
                          test_encodet,
                          _,
                      ) = self.gen_groupby_cat_encode_features(
                          data,
                          encodet_features_names,
                          num_col,
                          train_group_encoder,
                      )
                      if not isinstance(test_encodet, str):
                          test_data = pd.concat(
                              [test_data, test_encodet], axis=1
                          )
  
              if verbose > 0:
                  addl_features = data_encodet.shape[1] * len(
                      num_features
                  )
                  count_number_features += addl_features
                  print(
                      " + added ",
                      addl_features,
                      " Group-by Encoded Features using JamesSteinEncoder",
                  )
  
      # Drop source cat features
      if not len(cat_encoder_names) == 0:
          ### if there is no categorical encoding, then let the categorical_vars pass through.
          ### If they have been transformed into Cat Encoded variables, then you can drop them!
          data.drop(columns=encodet_features_names, inplace=True)
      # In this case, there may be some inf values, replace them ######
      data.replace([np.inf, -np.inf], np.nan, inplace=True)
      # data.fillna(0, inplace=True)
      if test_data is not None:
          if not len(cat_encoder_names) == 0:
              ### if there is no categorical encoding, then let the categorical_vars pass through.
              test_data.drop(
                  columns=encodet_features_names, inplace=True
              )
          test_data.replace([np.inf, -np.inf], np.nan, inplace=True)
          # test_data.fillna(0, inplace=True)
  
      X_train = copy.deepcopy(data)
      X_test = copy.deepcopy(test_data)
  
      # Normalization Data
      if normalization:
          if verbose > 0:
              print("> Normalization Features")
          columns_name = X_train.columns.values
          scaler = StandardScaler().fit(X_train)
          X_train = scaler.transform(X_train)
          X_test = scaler.transform(X_test)
          X_train = pd.DataFrame(X_train, columns=columns_name)
          X_test = pd.DataFrame(X_test, columns=columns_name)
  
      if verbose > 0:
          print("#" * 50)
          print("> Final Number of Features: ", (X_train.shape[1]))
          print("#" * 50)
          print(
              "New X_train rows: %s, X_test rows: %s"
              % (X_train.shape[0], X_test.shape[0])
          )
          print(
              "New X_train columns: %s, X_test columns: %s"
              % (X_train.shape[1], X_test.shape[1])
          )
          if len(left_subtract(X_test.columns, X_train.columns)) > 0:
              print(
                  """There are more columns in test than train 
                      due to missing columns being more in test than train. Continuing..."""
              )
  
      return X_train, X_test
  
imports: import pandas
  import numpy
  from itertools import combinations
  from sklearn.preprocessing import StandardScaler
  from category_encoders import HashingEncoder, SumEncoder, PolynomialEncoder, BackwardDifferenceEncoder
  from category_encoders import OneHotEncoder, HelmertEncoder, OrdinalEncoder, CountEncoder, BaseNEncoder
  from category_encoders import TargetEncoder, CatBoostEncoder, WOEEncoder, JamesSteinEncoder
  from category_encoders.glmm import GLMMEncoder
  from sklearn.preprocessing import LabelEncoder
  from category_encoders.wrapper import PolynomialWrapper
  from .encoders import FrequencyEncoder
  from .None import settings
  import pdb
  import copy
  import dask
  import dask.dataframe
label_desc: Use image node_id 9 for calling the DataBunch obj's underlying member method code with example usage: obj.preproc_data(X_train, X_test, y_train, cat_features, cat_encoder_names, clean_nan, num_generator_features, group_generator_features, target_enc_cat_features, normalization, verbose) and returns: X_train,  X_test