class_docstr: Test class for cross-validation tests.
class_docstr_tok: ["Test","class","for","cross-validation","tests","."]
docstr: Test cross-validation with custom specified seed.
  
  Test that cross-validation for either a classifier or a regressor
  with an custom seed and with grid search enabled generates the
  same folds as expected. We run the test using a configuration file
  as well as the API.
docstr_tok: ["Test","cross-validation","with","custom","specified","seed",".","Test","that","cross-validation","for","either","a","classifier","or","a","regressor","with","an","custom","seed","and","with","grid","search","enabled","generates","the","same","folds","as","expected",".","We","run","the","test","using","a","configuration","file","as","well","as","the","API","."]
code: def check_cross_validate_task_with_custom_seed(
      self, learner_type, use_config
  ):
      """
      Test cross-validation with custom specified seed.
  
      Test that cross-validation for either a classifier or a regressor
      with an custom seed and with grid search enabled generates the
      same folds as expected. We run the test using a configuration file
      as well as the API.
      """
      # load in the featureset
      suffix = ".jsonlines"
      train_path = train_dir / f"f0{suffix}"
      if learner_type == "classifier":
          examples = load_featureset(train_path, "", suffix, quiet=True)
      else:
          examples = load_featureset(
              train_path,
              "",
              suffix,
              class_map={"dog": 0, "cat": 1},
              quiet=True,
          )
  
      # use a configuration file or the API, as specified
      if use_config:
          template_name = (
              "test_custom_cv_seed_classifier.template.cfg"
              if learner_type == "classifier"
              else "test_custom_cv_seed_regressor.template.cfg"
          )
          template_path = config_dir / template_name
          config_path = fill_in_config_paths_for_single_file(
              template_path, train_path, None
          )
          run_configuration(config_path, quiet=True, local=True)
  
          # read in the folds file produced by SKLL
          computed_fold_ids = {}
          cv_folds_file_name = (
              "test_custom_cv_seed_clf_skll_fold_ids.csv"
              if learner_type == "classifier"
              else "test_custom_cv_seed_reg_skll_fold_ids.csv"
          )
          with open(output_dir / cv_folds_file_name) as f:
              reader = csv.DictReader(f)
              for row in reader:
                  computed_fold_ids[row["id"]] = row["cv_test_fold"]
  
      else:
          learner_name = (
              "LogisticRegression"
              if learner_type == "classifier"
              else "Ridge"
          )
          learner = Learner(learner_name)
          objective = (
              "accuracy" if learner_type == "classifier" else "pearson"
          )
          (_, _, _, computed_fold_ids, _) = learner.cross_validate(
              examples,
              cv_folds=10,
              cv_seed=54321,
              grid_search=True,
              grid_objective=objective,
          )
  
      # compute the fold IDs we expect from SKLL using the custom seed directly;
      # obviously, we only use stratified fold splitting with classifiers
      expected_fold_ids = compute_expected_folds_for_cv_testing(
          examples, stratified=learner_type == "classifier", seed=54321
      )
  
      # convert the dictionary to strings (sorted by key) for quick comparison
      computed_fold_ids_str = "".join(
          f"{key}{val}"
          for key, val in sorted(computed_fold_ids.items())
      )
      expected_fold_ids_str = "".join(
          f"{key}{val}"
          for key, val in sorted(expected_fold_ids.items())
      )
  
      # the two sets of folds must be equal
      self.assertEqual(computed_fold_ids_str, expected_fold_ids_str)
  
imports: import csv
  import itertools
  import json
  import re
  import unittest
  import numpy
  from numpy.testing import assert_almost_equal
  from sklearn.datasets import make_classification
  from sklearn.feature_extraction import FeatureHasher
  from skll.config import load_cv_folds
  from skll.data import FeatureSet
  from skll.experiments import load_featureset, run_configuration
  from skll.learner import Learner
  from skll.utils.testing import compute_expected_folds_for_cv_testing, config_dir, create_jsonlines_feature_files, fill_in_config_paths_for_single_file, other_dir, output_dir, remove_jsonlines_feature_files, train_dir, unlink
label_desc: Use image node_id 13 for calling the TestCrossValidation obj's underlying member method code with example usage: obj.check_cross_validate_task_with_custom_seed(learner_type, use_config) without return types