code: def testEvaluateWithCrossSlicing(self):
      temp_export_dir1 = self._getExportDir()
      (
          _,
          export_dir1,
      ) = fixed_prediction_estimator_extra_fields.simple_fixed_prediction_estimator_extra_fields(
          None, temp_export_dir1
      )
      temp_export_dir2 = self._getExportDir()
      (
          _,
          export_dir2,
      ) = fixed_prediction_estimator_extra_fields.simple_fixed_prediction_estimator_extra_fields(
          None, temp_export_dir2
      )
      example_count_metric = config_pb2.MetricConfig(
          class_name="ExampleCount",
          # 5 >= 3, OK for overall slice
          # 3 >= 3, OK for ('fixed_string', 'fixed_string1')
          # 2 < 3, NOT OK for ('fixed_string', 'fixed_string2')
          # Keep this for verifying cross slice thresholds and single slice
          # thresholds are working together.
          threshold=config_pb2.MetricThreshold(
              value_threshold=config_pb2.GenericValueThreshold(
                  lower_bound={"value": 3}
              )
          ),
          cross_slice_thresholds=[
              config_pb2.CrossSliceMetricThreshold(
                  # 5-2 >= 2, OK for ((), (('fixed_string', 'fixed_string1'),))
                  # 5-3 < 3, NOT OK for ((), (('fixed_string', 'fixed_string2'),))
                  threshold=config_pb2.MetricThreshold(
                      value_threshold=config_pb2.GenericValueThreshold(
                          lower_bound={"value": 3}
                      )
                  ),
                  cross_slicing_specs=[
                      config_pb2.CrossSlicingSpec(
                          baseline_spec=config_pb2.SlicingSpec(),
                          slicing_specs=[
                              config_pb2.SlicingSpec(
                                  feature_keys=["fixed_string"]
                              )
                          ],
                      )
                  ],
              )
          ],
      )
      mean_prediction_metric = config_pb2.MetricConfig(
          class_name="MeanPrediction",
          cross_slice_thresholds=[
              config_pb2.CrossSliceMetricThreshold(
                  # MeanPrediction values for slices:
                  # (0.2*2+0.9*2)/(2+2)=0.55
                  #     for (('fixed_string', 'fixed_string1'),)
                  # (0.5*2+0.5*2+0.5*2)/(2+2+2)=0.5
                  #     for (('fixed_string', 'fixed_string2'),)
                  threshold=config_pb2.MetricThreshold(
                      value_threshold=config_pb2.GenericValueThreshold(
                          # This config should give value threshold error because
                          # (0.55-0.5)=0.05 not inside the bound [0.1, 0.5].
                          upper_bound={"value": 0.5},
                          lower_bound={"value": 0.1},
                      ),
                      change_threshold=config_pb2.GenericChangeThreshold(
                          # This config should give change threshold error because
                          # baseline model and candidate model have same
                          # difference as 0.05 between cross slices. Cross slice
                          # difference value is not changed.
                          direction=config_pb2.MetricDirection.LOWER_IS_BETTER,
                          relative={"value": -0.99},
                          absolute={"value": 0},
                      ),
                  ),
                  cross_slicing_specs=[
                      config_pb2.CrossSlicingSpec(
                          baseline_spec=config_pb2.SlicingSpec(
                              feature_values={
                                  "fixed_string": "fixed_string1"
                              }
                          ),
                          slicing_specs=[
                              config_pb2.SlicingSpec(
                                  feature_values={
                                      "fixed_string": "fixed_string2"
                                  }
                              )
                          ],
                      )
                  ],
              )
          ],
      )
      eval_config = config_pb2.EvalConfig(
          model_specs=[
              config_pb2.ModelSpec(
                  name="candidate",
                  label_key="label",
                  example_weight_key="fixed_float",
              ),
              config_pb2.ModelSpec(
                  name="baseline",
                  label_key="label",
                  example_weight_key="fixed_float",
                  is_baseline=True,
              ),
          ],
          slicing_specs=[
              config_pb2.SlicingSpec(),
              config_pb2.SlicingSpec(feature_keys=["fixed_string"]),
          ],
          cross_slicing_specs=[
              config_pb2.CrossSlicingSpec(
                  baseline_spec=config_pb2.SlicingSpec(),
                  slicing_specs=[
                      config_pb2.SlicingSpec(
                          feature_keys=["fixed_string"]
                      )
                  ],
              ),
              config_pb2.CrossSlicingSpec(
                  baseline_spec=config_pb2.SlicingSpec(
                      feature_values={"fixed_string": "fixed_string1"}
                  ),
                  slicing_specs=[
                      config_pb2.SlicingSpec(
                          feature_values={
                              "fixed_string": "fixed_string2"
                          }
                      )
                  ],
              ),
          ],
          metrics_specs=[
              config_pb2.MetricsSpec(
                  metrics=[example_count_metric],
                  model_names=["candidate", "baseline"],
                  example_weights=config_pb2.ExampleWeightOptions(
                      unweighted=True
                  ),
              ),
              config_pb2.MetricsSpec(
                  metrics=[mean_prediction_metric],
                  model_names=["candidate", "baseline"],
              ),
          ],
      )
      eval_shared_model = [
          self.createTestEvalSharedModel(
              model_name="candidate", eval_saved_model_path=export_dir1
          ),
          self.createTestEvalSharedModel(
              model_name="baseline", eval_saved_model_path=export_dir2
          ),
      ]
      extractors = [
          legacy_predict_extractor.PredictExtractor(
              eval_shared_model=eval_shared_model,
              eval_config=eval_config,
          ),
          unbatch_extractor.UnbatchExtractor(),
          slice_key_extractor.SliceKeyExtractor(
              eval_config=eval_config
          ),
      ]
      evaluators = [
          metrics_plots_and_validations_evaluator.MetricsPlotsAndValidationsEvaluator(
              eval_config=eval_config,
              eval_shared_model=eval_shared_model,
          )
      ]
  
      # fixed_float used as example_weight key
      examples = [
          self._makeExample(
              prediction=0.2,
              label=1.0,
              fixed_int=1,
              fixed_float=1.0,
              fixed_string="fixed_string1",
          ),
          self._makeExample(
              prediction=0.9,
              label=0.0,
              fixed_int=1,
              fixed_float=1.0,
              fixed_string="fixed_string1",
          ),
          self._makeExample(
              prediction=0.5,
              label=0.0,
              fixed_int=2,
              fixed_float=2.0,
              fixed_string="fixed_string2",
          ),
          self._makeExample(
              prediction=0.5,
              label=0.0,
              fixed_int=2,
              fixed_float=2.0,
              fixed_string="fixed_string2",
          ),
          self._makeExample(
              prediction=0.5,
              label=0.0,
              fixed_int=2,
              fixed_float=2.0,
              fixed_string="fixed_string2",
          ),
      ]
  
      tfx_io = raw_tf_record.RawBeamRecordTFXIO(
          physical_format="inmemory",
          raw_record_column_name=constants.ARROW_INPUT_COLUMN,
          telemetry_descriptors=["TFMATest"],
      )
      with beam.Pipeline() as pipeline:
          # pylint: disable=no-value-for-parameter
          evaluations = (
              pipeline
              | "Create"
              >> beam.Create([e.SerializeToString() for e in examples])
              | "BatchExamples" >> tfx_io.BeamSource()
              | "InputsToExtracts"
              >> model_eval_lib.BatchedInputsToExtracts()
              | "ExtractAndEvaluate"
              >> model_eval_lib.ExtractAndEvaluate(
                  extractors=extractors, evaluators=evaluators
              )
          )
  
          # pylint: enable=no-value-for-parameter
  
          def check_validations(got):
              try:
                  self.assertLen(got, 6)
  
                  def get_slice_keys_hash(metric_validation):
                      slice_key, cross_slice_key = None, None
                      if metric_validation.slice_key:
                          slice_key = (
                              metric_validation.slice_key.SerializeToString()
                          )
                      if metric_validation.cross_slice_key:
                          cross_slice_key = (
                              metric_validation.cross_slice_key.SerializeToString()
                          )
                      return (slice_key, cross_slice_key)
  
                  successful_validations = []
                  failed_validations = {}
                  for validation_result in got:
                      if validation_result.validation_ok:
                          successful_validations.append(
                              validation_result
                          )
                      else:
                          self.assertLen(
                              validation_result.metric_validations_per_slice,
                              1,
                          )
                          validation_result = validation_result.metric_validations_per_slice[
                              0
                          ]
                          slice_keys_hash = get_slice_keys_hash(
                              validation_result
                          )
                          failed_validations[slice_keys_hash] = {}
                          for failure in validation_result.failures:
                              failure.ClearField("metric_value")
                              failed_validations[slice_keys_hash][
                                  failure.metric_key.SerializeToString()
                              ] = failure
                  self.assertLen(successful_validations, 3)
                  self.assertLen(failed_validations, 3)
  
                  expected_validations = [
                      text_format.Parse(
                          """
                    slice_key {
                      single_slice_keys {
                        column: "fixed_string"
                        bytes_value: "fixed_string1"
                      }
                    }
                    failures {
                      metric_key {
                        name: "example_count"
                        model_name: "candidate"
                        example_weighted { }
                      }
                      metric_threshold {
                        value_threshold {
                          lower_bound {
                            value: 3.0
                          }
                        }
                      }
                    }
                    """,
                          validation_result_pb2.MetricsValidationForSlice(),
                      ),
                      text_format.Parse(
                          """
                    failures {
                      metric_key {
                        name: "example_count"
                        model_name: "candidate"
                        example_weighted { }
                      }
                      metric_threshold {
                        value_threshold {
                          lower_bound {
                            value: 3.0
                          }
                        }
                      }
                    }
                    cross_slice_key {
                      baseline_slice_key {
                      }
                      comparison_slice_key {
                        single_slice_keys {
                          column: "fixed_string"
                          bytes_value: "fixed_string2"
                        }
                      }
                    }
                    """,
                          validation_result_pb2.MetricsValidationForSlice(),
                      ),
                      text_format.Parse(
                          """
                    failures {
                      metric_key {
                        name: "mean_prediction"
                        model_name: "candidate"
                        example_weighted { value: true }
                      }
                      metric_threshold {
                        value_threshold {
                          lower_bound {
                            value: 0.1
                          }
                          upper_bound {
                            value: 0.5
                          }
                        }
                      }
                    }
                    failures {
                      metric_key {
                        name: "mean_prediction"
                        model_name: "candidate"
                        is_diff: true
                        example_weighted { value: true }
                      }
                      metric_threshold {
                        change_threshold {
                          absolute {
                          }
                          relative {
                            value: -0.99
                          }
                          direction: LOWER_IS_BETTER
                        }
                      }
                    }
                    cross_slice_key {
                      baseline_slice_key {
                        single_slice_keys {
                          column: "fixed_string"
                          bytes_value: "fixed_string1"
                        }
                      }
                      comparison_slice_key {
                        single_slice_keys {
                          column: "fixed_string"
                          bytes_value: "fixed_string2"
                        }
                      }
                    }
                    """,
                          validation_result_pb2.MetricsValidationForSlice(),
                      ),
                  ]
  
                  expected_validations_dict = {}
                  for expected_validation in expected_validations:
                      slice_keys_hash = get_slice_keys_hash(
                          expected_validation
                      )
                      expected_validations_dict[slice_keys_hash] = {}
                      for failure in expected_validation.failures:
                          expected_validations_dict[slice_keys_hash][
                              failure.metric_key.SerializeToString()
                          ] = failure
  
                  self.assertEqual(
                      set(failed_validations),
                      set(expected_validations_dict),
                  )
                  for (
                      slice_key,
                      validation,
                  ) in failed_validations.items():
                      self.assertEqual(
                          set(validation),
                          set(expected_validations_dict[slice_key]),
                      )
                      for metric_key, failure in validation.items():
                          self.assertProtoEquals(
                              failure,
                              expected_validations_dict[slice_key][
                                  metric_key
                              ],
                          )
              except AssertionError as err:
                  raise util.BeamAssertException(err)
  
          util.assert_that(
              evaluations[constants.VALIDATIONS_KEY], check_validations
          )
  
imports: import os
  from absl.testing import parameterized
  import apache_beam
  from apache_beam.testing import util
  import numpy
  import tensorflow
  from tensorflow_model_analysis import constants
  from tensorflow_model_analysis.addons.fairness.metrics import lift
  from tensorflow_model_analysis.api import model_eval_lib
  from tensorflow_model_analysis.eval_saved_model import testutil
  from tensorflow_model_analysis.eval_saved_model.example_trainers import dnn_classifier
  from tensorflow_model_analysis.eval_saved_model.example_trainers import fixed_prediction_estimator_extra_fields
  from tensorflow_model_analysis.eval_saved_model.example_trainers import linear_classifier
  from tensorflow_model_analysis.eval_saved_model.example_trainers import multi_head
  from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
  from tensorflow_model_analysis.extractors import example_weights_extractor
  from tensorflow_model_analysis.extractors import features_extractor
  from tensorflow_model_analysis.extractors import labels_extractor
  from tensorflow_model_analysis.extractors import legacy_predict_extractor
  from tensorflow_model_analysis.extractors import materialized_predictions_extractor
  from tensorflow_model_analysis.extractors import predictions_extractor
  from tensorflow_model_analysis.extractors import slice_key_extractor
  from tensorflow_model_analysis.extractors import unbatch_extractor
  from tensorflow_model_analysis.metrics import attributions
  from tensorflow_model_analysis.metrics import binary_confusion_matrices
  from tensorflow_model_analysis.metrics import calibration
  from tensorflow_model_analysis.metrics import calibration_plot
  from tensorflow_model_analysis.metrics import confusion_matrix_plot
  from tensorflow_model_analysis.metrics import metric_specs
  from tensorflow_model_analysis.metrics import metric_types
  from tensorflow_model_analysis.metrics import ndcg
  from tensorflow_model_analysis.post_export_metrics import metrics
  from tensorflow_model_analysis.proto import config_pb2
  from tensorflow_model_analysis.proto import validation_result_pb2
  from tfx_bsl.tfxio import raw_tf_record
  from tfx_bsl.tfxio import tensor_adapter
  from tfx_bsl.tfxio import test_util
  from google.protobuf import text_format
  from tensorflow_metadata.proto.v0 import schema_pb2
label_desc: Use image node_id 22 for calling the MetricsPlotsAndValidationsEvaluatorTest obj's underlying member method code with example usage: obj.testEvaluateWithCrossSlicing() and returns: slice_key,  cross_slice_key