code: def test_model_forecast_evaluation(df):
      """Tests whether model, forecast, evaluation parameters are properly set in pipeline:
  
      Parameters tested:
  
          # model
          - pipeline
          - estimator_name
          - hyperparameter_grid
          - hyperparameter_budget
          - n_jobs
          - verbose
          # forecast
          - forecast_horizon
          - coverage
          - agg_periods
          - agg_func
          # evaluation
          - score_func
          - score_func_greater_is_better
          - cv_report_metrics
          - null_model_params
          - relative_error_tolerance
      """
      # Checks that grid search scores are as expected.
      # 4 options in the grid, score_func_greater_is_better=True
      hyperparameter_grid = [
          {
              "estimator__strategy": ["quantile"],
              "estimator__quantile": [0.9],
          },
          {"estimator__strategy": ["mean"]},
          {
              "estimator__strategy": ["constant"],
              "estimator__constant": [1.0, 2.0],
          },
      ]
      pipeline = get_dummy_pipeline()
      with warnings.catch_warnings():
          warnings.simplefilter("ignore")
          metric = EvaluationMetricEnum.MeanAbsolutePercentError
          result = forecast_pipeline(
              df,
              pipeline=pipeline,
              hyperparameter_grid=hyperparameter_grid,
              hyperparameter_budget=None,  # full grid search
              n_jobs=-1,
              verbose=2,
              forecast_horizon=20,
              coverage=None,
              agg_periods=7,
              agg_func=np.sum,
              score_func=metric.name,
              score_func_greater_is_better=metric.get_metric_greater_is_better(),
              cv_report_metrics=CV_REPORT_METRICS_ALL,
              null_model_params=None,
              relative_error_tolerance=0.02,
          )
          expected_report_metrics = [
              enum.get_metric_name() for enum in EvaluationMetricEnum
          ]
          expected_report_metrics += [FRACTION_OUTSIDE_TOLERANCE_NAME]
          check_forecast_pipeline_result(
              result,
              coverage=None,
              expected_grid_size=4,
              lower_bound_cv=0.0,  # MAPE is non-negative
              score_func=metric.name,
              greater_is_better=metric.get_metric_greater_is_better(),
              cv_report_metrics_names=expected_report_metrics,
              relative_error_tolerance=0.02,
          )
          df_train = result.backtest.df_train
          df_test = result.backtest.df_test
          for enum in EvaluationMetricEnum:
              scorer = enum.get_metric_func()
              # Note: `agg_func` is used in CV evaluation but currently not used
              #   in reporting backtest/forecast metrics. Check when this is enabled.
              # scorer = add_preaggregation_to_scorer(scorer, agg_periods=7, agg_func=np.sum)
              expected_score = scorer(
                  df_train[ACTUAL_COL], df_train[PREDICTED_COL]
              )  # train score
              assert (
                  result.backtest.train_evaluation[
                      enum.get_metric_name()
                  ]
                  == expected_score
              )
              expected_score = scorer(
                  df_test[ACTUAL_COL], df_test[PREDICTED_COL]
              )  # test score
              assert (
                  result.backtest.test_evaluation[
                      enum.get_metric_name()
                  ]
                  == expected_score
              )
          assert (
              result.backtest.test_evaluation[
                  FRACTION_OUTSIDE_TOLERANCE
              ]
              == result.backtest.test_evaluation[
                  EvaluationMetricEnum.FractionOutsideTolerance2.get_metric_name()
              ]
          )
          assert (
              result.forecast.train_evaluation[
                  FRACTION_OUTSIDE_TOLERANCE
              ]
              == result.forecast.train_evaluation[
                  EvaluationMetricEnum.FractionOutsideTolerance2.get_metric_name()
              ]
          )
      assert len(result.grid_search.cv_results_.keys()) == 206
  
      # Tests estimator_name, coverage, null_model_params,
      # score_func callable, score_func_greater_is_better=True
      with warnings.catch_warnings():
          warnings.simplefilter("ignore")
          hyperparameter_grid = {
              "estimator__fit_algorithm_dict": [
                  {
                      "fit_algorithm": "linear",
                      "fit_algorithm_params": {"fit_intercept": False},
                  },
                  {
                      "fit_algorithm": "ridge",
                  },
              ],
              "estimator__extra_pred_cols": [
                  ["ct1", "is_weekend"],
                  ["ct1"],
                  ["ct2"],
              ],
          }
          null_model_params = {"strategy": "mean"}
          coverage = 0.80
          hyperparameter_budget = 2  # limited grid search
          result = forecast_pipeline(
              df,
              estimator=SilverkiteEstimator(),
              hyperparameter_grid=hyperparameter_grid,
              hyperparameter_budget=hyperparameter_budget,
              n_jobs=1,
              verbose=1,
              forecast_horizon=10,
              coverage=coverage,
              agg_periods=2,
              agg_func=np.max,
              score_func=explained_variance_score,
              score_func_greater_is_better=True,
              cv_report_metrics=None,
              null_model_params=null_model_params,
              relative_error_tolerance=None,
              cv_max_splits=1,
          )
          check_forecast_pipeline_result(
              result,
              coverage=coverage,
              strategy=null_model_params["strategy"],
              expected_grid_size=hyperparameter_budget,
              upper_bound_cv=1.0,  # highest possible explained variance
              score_func=explained_variance_score,
              greater_is_better=True,
              cv_report_metrics_names=None,
              relative_error_tolerance=None,
          )
          # Fewer metrics than before, because cv_report_metrics=None
          assert len(result.grid_search.cv_results_.keys()) == 14
          assert (
              "mean_test_CORR"
              not in result.grid_search.cv_results_.keys()
          )
          assert R2_null_model_score in result.backtest.test_evaluation
  
      # Tests estimator_name, coverage, score_func=FRACTION_OUTSIDE_TOLERANCE
      with warnings.catch_warnings():
          warnings.simplefilter("ignore")
          result = forecast_pipeline(
              df,
              estimator=SilverkiteEstimator(),
              forecast_horizon=10,
              coverage=coverage,
              agg_periods=2,
              agg_func=np.max,
              score_func=FRACTION_OUTSIDE_TOLERANCE,
              score_func_greater_is_better=False,
              relative_error_tolerance=0.02,
              cv_max_splits=1,
          )
          assert (
              f"mean_test_{FRACTION_OUTSIDE_TOLERANCE_NAME}"
              in result.grid_search.cv_results_.keys()
          )
  
imports: import datetime
  import math
  import sys
  import warnings
  import numpy
  import pandas
  import pytest
  from scipy.stats import randint
  from sklearn.metrics import explained_variance_score
  from sklearn.metrics import mean_absolute_error
  from sklearn.metrics import mean_squared_error
  from sklearn.pipeline import Pipeline
  from greykite.common.constants import ACTUAL_COL
  from greykite.common.constants import ADJUSTMENT_DELTA_COL
  from greykite.common.constants import END_TIME_COL
  from greykite.common.constants import FRACTION_OUTSIDE_TOLERANCE
  from greykite.common.constants import METRIC_COL
  from greykite.common.constants import PREDICTED_COL
  from greykite.common.constants import QUANTILE_SUMMARY_COL
  from greykite.common.constants import START_TIME_COL
  from greykite.common.constants import TIME_COL
  from greykite.common.constants import VALUE_COL
  from greykite.common.constants import R2_null_model_score
  from greykite.common.evaluation import EvaluationMetricEnum
  from greykite.common.evaluation import add_finite_filter_to_scorer
  from greykite.common.evaluation import add_preaggregation_to_scorer
  from greykite.common.python_utils import assert_equal
  from greykite.common.python_utils import unique_elements_in_list
  from greykite.common.testing_utils import generate_df_for_tests
  from greykite.common.testing_utils import generate_df_with_reg_for_tests
  from greykite.framework.constants import CV_REPORT_METRICS_ALL
  from greykite.framework.constants import FRACTION_OUTSIDE_TOLERANCE_NAME
  from greykite.framework.input.univariate_time_series import UnivariateTimeSeries
  from greykite.framework.pipeline.pipeline import forecast_pipeline
  from greykite.framework.templates.prophet_template import ProphetTemplate
  from greykite.framework.utils.framework_testing_utils import check_forecast_pipeline_result
  from greykite.framework.utils.framework_testing_utils import mock_pipeline
  from greykite.sklearn.estimator.null_model import DummyEstimator
  from greykite.sklearn.estimator.prophet_estimator import ProphetEstimator
  from greykite.sklearn.estimator.silverkite_estimator import SilverkiteEstimator
  from greykite.sklearn.estimator.simple_silverkite_estimator import SimpleSilverkiteEstimator
  from greykite.sklearn.transform.column_selector import ColumnSelector
  from greykite.sklearn.transform.drop_degenerate_transformer import DropDegenerateTransformer
  from greykite.sklearn.transform.dtype_column_selector import DtypeColumnSelector
  from greykite.sklearn.transform.normalize_transformer import NormalizeTransformer
  from greykite.sklearn.transform.null_transformer import NullTransformer
  from greykite.sklearn.transform.pandas_feature_union import PandasFeatureUnion
  from greykite.sklearn.transform.zscore_outlier_transformer import ZscoreOutlierTransformer
label_desc: Use image node_id 11 for calling a global function with example usage: test_model_forecast_evaluation(df) without return types