Updated results

Signed-off-by: Jonathan Bnayahu <[email protected]>

results/bluebench/{2025-07-02T14-58-20_evaluation_results.json → 2025-07-03T07-20-08_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T18:58:17.004768Z",
+        "timestamp_utc": "2025-07-03T11:20:04.599853Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,16 +176,6 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.6666666666666666,
-                "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.0,
-                "num_of_instances": 9
-            },
-            "safety_bbq_disability_status": {
                 "accuracy": 0.5555555555555556,
                 "accuracy_ci_low": 0.2222222222222222,
                 "accuracy_ci_high": 0.8888888888888888,
@@ -195,14 +185,24 @@
                 "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.3333333333333333,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.6666666666666666,
+                "score_name": "accuracy",
+                "score": 0.3333333333333333,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.4444444444444444,
+                "score_ci_low": 0.5555555555555556,
                 "num_of_instances": 9
             },
             "safety_bbq_nationality": {
@@ -236,16 +236,6 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_gender": {
-                "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.2222222222222222,
-                "accuracy_ci_high": 0.8888888888888888,
-                "score_name": "accuracy",
-                "score": 0.5555555555555556,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.2222222222222222,
-                "num_of_instances": 9
-            },
-            "safety_bbq_race_x_ses": {
                 "accuracy": 0.6666666666666666,
                 "accuracy_ci_low": 0.3333333333333333,
                 "accuracy_ci_high": 0.8888888888888888,
@@ -255,6 +245,16 @@
                 "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
+                "score_name": "accuracy",
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
             "safety_bbq_religion": {
                 "accuracy": 0.4444444444444444,
                 "accuracy_ci_low": 0.1111111111111111,
@@ -266,61 +266,61 @@
                 "num_of_instances": 9
             },
             "safety_bbq_ses": {
-                "accuracy": 0.2222222222222222,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5555555555555556,
+                "accuracy": 0.3333333333333333,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.6666666666666666,
                 "score_name": "accuracy",
-                "score": 0.2222222222222222,
-                "score_ci_high": 0.5555555555555556,
-                "score_ci_low": 0.0,
+                "score": 0.3333333333333333,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.1111111111111111,
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
                 "accuracy": 0.2222222222222222,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5780215743718348,
+                "accuracy_ci_high": 0.6666666666666666,
                 "score_name": "accuracy",
                 "score": 0.2222222222222222,
-                "score_ci_high": 0.5780215743718348,
+                "score_ci_high": 0.6666666666666666,
                 "score_ci_low": 0.0,
                 "num_of_instances": 9
             },
-            "score": 0.45454545454545453,
+            "score": 0.46464646464646464,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.31693989071038253,
-                "score": 0.31693989071038253,
+                "llama_3_70b_instruct_template_arena_hard": 0.30994152046783624,
+                "score": 0.30994152046783624,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.31693989071038253,
+            "score": 0.30994152046783624,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.19354838709677416,
-                "f1_Organization": 0.3111111111111111,
-                "f1_Location": 0.11764705882352941,
-                "f1_macro": 0.20743551901047155,
-                "recall_macro": 0.15458937198067632,
-                "precision_macro": 0.328921568627451,
-                "in_classes_support": 0.7,
-                "f1_micro": 0.19199999999999998,
-                "recall_micro": 0.16,
-                "precision_micro": 0.24,
-                "score": 0.19199999999999998,
+                "f1_Person": 0.4571428571428571,
+                "f1_Organization": 0.24561403508771928,
+                "f1_Location": 0.2727272727272727,
+                "f1_macro": 0.32516138831928304,
+                "recall_macro": 0.2826086956521739,
+                "precision_macro": 0.40268199233716473,
+                "in_classes_support": 0.7349397590361446,
+                "f1_micro": 0.26582278481012656,
+                "recall_micro": 0.28,
+                "precision_micro": 0.25301204819277107,
+                "score": 0.26582278481012656,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.10104164340624598,
-                "score_ci_high": 0.27209420121438893,
-                "f1_micro_ci_low": 0.10104164340624598,
-                "f1_micro_ci_high": 0.27209420121438893
+                "score_ci_low": 0.1803099177588422,
+                "score_ci_high": 0.37307865079917296,
+                "f1_micro_ci_low": 0.1803099177588422,
+                "f1_micro_ci_high": 0.37307865079917296
             },
-            "score": 0.19199999999999998,
+            "score": 0.26582278481012656,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -338,21 +338,21 @@
             "mmlu_pro_business": {
                 "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7469722493882013,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
                 "score": 0.14285714285714285,
-                "score_ci_high": 0.7469722493882013,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
@@ -386,22 +386,22 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.0,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -436,22 +436,22 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.6807203593841678,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.6807203593841678,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -465,38 +465,38 @@
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
-            "score": 0.26530612244897955,
+            "score": 0.29591836734693877,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.3833333333333333,
+                "f1_macro": 0.19047619047619047,
                 "f1_suggestive": 0.0,
-                "f1_arbitrary": 0.5,
+                "f1_fanciful": 0.2857142857142857,
                 "f1_generic": 0.0,
-                "f1_descriptive": 0.75,
-                "f1_fanciful": 0.6666666666666666,
-                "f1_macro_ci_low": 0.23401046490188043,
-                "f1_macro_ci_high": 0.5639947336385638,
+                "f1_arbitrary": 0.0,
+                "f1_descriptive": 0.6666666666666666,
+                "f1_macro_ci_low": 0.030903392316786366,
+                "f1_macro_ci_high": 0.3142857142857143,
                 "score_name": "f1_micro",
-                "score": 0.45714285714285713,
-                "score_ci_high": 0.7027027027027027,
-                "score_ci_low": 0.24242424242424243,
+                "score": 0.21052631578947367,
+                "score_ci_high": 0.42424242424242425,
+                "score_ci_low": 0.05263157894736842,
                 "num_of_instances": 20,
-                "accuracy": 0.4,
-                "accuracy_ci_low": 0.2,
-                "accuracy_ci_high": 0.65,
-                "f1_micro": 0.45714285714285713,
-                "f1_micro_ci_low": 0.24242424242424243,
-                "f1_micro_ci_high": 0.7027027027027027
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.21052631578947367,
+                "f1_micro_ci_low": 0.05263157894736842,
+                "f1_micro_ci_high": 0.42424242424242425
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.4982078853046595,
-                "f1_no": 0.7741935483870968,
-                "f1_yes": 0.2222222222222222,
-                "f1_macro_ci_low": 0.3548387096774194,
-                "f1_macro_ci_high": 0.8838212196875215,
+                "f1_macro": 0.561128526645768,
+                "f1_no": 0.7586206896551724,
+                "f1_yes": 0.36363636363636365,
+                "f1_macro_ci_low": 0.3732193732193732,
+                "f1_macro_ci_high": 0.8986345790442171,
                 "score_name": "f1_micro",
                 "score": 0.65,
                 "score_ci_high": 0.85,
@@ -510,203 +510,203 @@
                 "f1_micro_ci_high": 0.85
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.11355311355311357,
-                "f1_conclusion": 0.3333333333333333,
-                "f1_rule": 0.0,
-                "f1_decree": 0.0,
+                "f1_macro": 0.12380952380952381,
+                "f1_conclusion": 0.0,
+                "f1_issue": 0.5333333333333333,
+                "f1_decree": 0.3333333333333333,
                 "f1_analysis": 0.0,
-                "f1_issue": 0.46153846153846156,
-                "f1_procedural history": 0.0,
                 "f1_facts": 0.0,
-                "f1_macro_ci_low": 0.03031798789203245,
-                "f1_macro_ci_high": 0.24492716802205583,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.04476661678289675,
+                "f1_macro_ci_high": 0.2683311877971836,
                 "score_name": "f1_micro",
-                "score": 0.20512820512820512,
-                "score_ci_high": 0.42105263157894735,
-                "score_ci_low": 0.05263157894736842,
+                "score": 0.2631578947368421,
+                "score_ci_high": 0.5,
+                "score_ci_low": 0.10256410256410256,
                 "num_of_instances": 20,
-                "accuracy": 0.2,
-                "accuracy_ci_low": 0.05,
-                "accuracy_ci_high": 0.4,
-                "f1_micro": 0.20512820512820512,
-                "f1_micro_ci_low": 0.05263157894736842,
-                "f1_micro_ci_high": 0.42105263157894735
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.2631578947368421,
+                "f1_micro_ci_low": 0.10256410256410256,
+                "f1_micro_ci_high": 0.5
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5333333333333333,
-                "f1_yes": 0.6666666666666666,
-                "f1_no": 0.4,
-                "f1_macro_ci_low": 0.32142857142857145,
-                "f1_macro_ci_high": 0.7684210526315789,
+                "f1_macro": 0.5604395604395604,
+                "f1_yes": 0.6923076923076923,
+                "f1_no": 0.42857142857142855,
+                "f1_macro_ci_low": 0.34065934065934067,
+                "f1_macro_ci_high": 0.7916666666666667,
                 "score_name": "f1_micro",
-                "score": 0.5641025641025641,
-                "score_ci_high": 0.7692307692307693,
-                "score_ci_low": 0.34978373445915895,
+                "score": 0.6,
+                "score_ci_high": 0.8,
+                "score_ci_low": 0.35,
                 "num_of_instances": 20,
-                "accuracy": 0.55,
-                "accuracy_ci_low": 0.3199652777510431,
-                "accuracy_ci_high": 0.75,
-                "f1_micro": 0.5641025641025641,
-                "f1_micro_ci_low": 0.34978373445915895,
-                "f1_micro_ci_high": 0.7692307692307693
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.35,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.6,
+                "f1_micro_ci_low": 0.35,
+                "f1_micro_ci_high": 0.8
             },
             "legalbench_proa": {
-                "f1_macro": 0.8174603174603174,
-                "f1_yes": 0.8571428571428571,
-                "f1_no": 0.7777777777777778,
-                "f1_macro_ci_low": 0.6114676143429352,
-                "f1_macro_ci_high": 0.949874686716792,
+                "f1_macro": 0.7333333333333334,
+                "f1_yes": 0.6666666666666666,
+                "f1_no": 0.8,
+                "f1_macro_ci_low": 0.500669556931299,
+                "f1_macro_ci_high": 0.9085714285714286,
                 "score_name": "f1_micro",
-                "score": 0.8205128205128205,
-                "score_ci_high": 0.95,
-                "score_ci_low": 0.6153846153846154,
+                "score": 0.7368421052631579,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.5,
                 "num_of_instances": 20,
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.6,
-                "accuracy_ci_high": 0.95,
-                "f1_micro": 0.8205128205128205,
-                "f1_micro_ci_low": 0.6153846153846154,
-                "f1_micro_ci_high": 0.95
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7368421052631579,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.9
             },
-            "score": 0.5393772893772893,
+            "score": 0.4921052631578947,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.30137445887445885,
-                "f1_cars": 0.7272727272727273,
+                "f1_macro": 0.3119817927170868,
+                "f1_cars": 0.8888888888888888,
                 "f1_windows x": 0.0,
                 "f1_atheism": 0.0,
-                "f1_christianity": 0.5714285714285714,
+                "f1_christianity": 0.0,
                 "f1_religion": 0.0,
-                "f1_medicine": 0.0,
-                "f1_computer graphics": 0.5714285714285714,
-                "f1_microsoft windows": 0.5,
+                "f1_medicine": 0.4,
+                "f1_computer graphics": 0.16666666666666666,
+                "f1_pc hardware": 0.47058823529411764,
+                "f1_cryptography": 0.6,
+                "f1_microsoft windows": 0.0,
                 "f1_middle east": 0.0,
-                "f1_politics": 0.2857142857142857,
-                "f1_motorcycles": 0.25,
-                "f1_mac hardware": 0.2857142857142857,
-                "f1_pc hardware": 0.26666666666666666,
-                "f1_electronics": 0.5,
+                "f1_politics": 0.4,
+                "f1_motorcycles": 0.4444444444444444,
+                "f1_baseball": 0.5,
+                "f1_mac hardware": 0.3333333333333333,
                 "f1_for sale": 0.3333333333333333,
                 "f1_guns": 0.0,
-                "f1_space": 0.5714285714285714,
-                "f1_cryptography": 0.2857142857142857,
-                "f1_baseball": 0.5454545454545454,
-                "f1_hockey": 0.3333333333333333,
-                "f1_macro_ci_low": 0.23918926535588275,
-                "f1_macro_ci_high": 0.40599488243071413,
+                "f1_space": 0.2857142857142857,
+                "f1_electronics": 0.6666666666666666,
+                "f1_hockey": 0.75,
+                "f1_macro_ci_low": 0.25044781739741556,
+                "f1_macro_ci_high": 0.4104329768429988,
                 "score_name": "f1_micro",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.43373493975903615,
-                "score_ci_low": 0.2360248447204969,
+                "score": 0.35802469135802467,
+                "score_ci_high": 0.4639265152043833,
+                "score_ci_low": 0.25928805281404416,
                 "num_of_instances": 100,
-                "accuracy": 0.27,
-                "accuracy_ci_low": 0.19,
-                "accuracy_ci_high": 0.36,
-                "f1_micro": 0.3333333333333333,
-                "f1_micro_ci_low": 0.2360248447204969,
-                "f1_micro_ci_high": 0.43373493975903615
+                "accuracy": 0.29,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.3830447129752326,
+                "f1_micro": 0.35802469135802467,
+                "f1_micro_ci_low": 0.25928805281404416,
+                "f1_micro_ci_high": 0.4639265152043833
             },
-            "score": 0.3333333333333333,
+            "score": 0.35802469135802467,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.5896499490484451,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9172932330827067,
-                "f1_credit card or prepaid card": 0.3076923076923077,
-                "f1_money transfer or virtual currency or money service": 0.8,
+                "f1_macro": 0.6884203769587696,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8854961832061069,
+                "f1_debt collection": 0.7058823529411765,
+                "f1_money transfer or virtual currency or money service": 0.5,
                 "f1_mortgage": 0.6666666666666666,
-                "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_debt collection": 0.6666666666666666,
+                "f1_credit card or prepaid card": 0.625,
                 "f1_checking or savings account": 0.7692307692307693,
-                "f1_macro_ci_low": 0.3258080885737301,
-                "f1_macro_ci_high": 0.7354751788733711,
+                "f1_payday loan or title loan or personal loan": 0.6666666666666666,
+                "f1_macro_ci_low": 0.5418497377329817,
+                "f1_macro_ci_high": 0.8476562978403089,
                 "score_name": "f1_micro",
-                "score": 0.8172043010752689,
-                "score_ci_high": 0.8756756756756757,
-                "score_ci_low": 0.7191151800610749,
+                "score": 0.8235294117647058,
+                "score_ci_high": 0.8864134451087434,
+                "score_ci_low": 0.7374301675977654,
                 "num_of_instances": 100,
-                "accuracy": 0.76,
+                "accuracy": 0.77,
                 "accuracy_ci_low": 0.67,
                 "accuracy_ci_high": 0.85,
-                "f1_micro": 0.8172043010752689,
-                "f1_micro_ci_low": 0.7191151800610749,
-                "f1_micro_ci_high": 0.8756756756756757
+                "f1_micro": 0.8235294117647058,
+                "f1_micro_ci_low": 0.7374301675977654,
+                "f1_micro_ci_high": 0.8864134451087434
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.5410602593440121,
-                "f1_mortgages and loans": 0.7,
-                "f1_credit card": 0.5333333333333333,
-                "f1_debt collection": 0.5263157894736842,
-                "f1_credit reporting": 0.6956521739130435,
-                "f1_retail banking": 0.25,
-                "f1_macro_ci_low": 0.42536480028629614,
-                "f1_macro_ci_high": 0.724652829691291,
+                "f1_macro": 0.5722222222222222,
+                "f1_mortgages and loans": 0.5833333333333334,
+                "f1_credit card": 0.5,
+                "f1_debt collection": 0.6666666666666666,
+                "f1_credit reporting": 0.6666666666666666,
+                "f1_retail banking": 0.4444444444444444,
+                "f1_macro_ci_low": 0.4373897842362741,
+                "f1_macro_ci_high": 0.730682215653928,
                 "score_name": "f1_micro",
-                "score": 0.5882352941176471,
-                "score_ci_high": 0.7143825412589823,
-                "score_ci_low": 0.4444444444444444,
+                "score": 0.5909090909090909,
+                "score_ci_high": 0.723404255319149,
+                "score_ci_low": 0.449438202247191,
                 "num_of_instances": 50,
-                "accuracy": 0.5,
-                "accuracy_ci_low": 0.36,
-                "accuracy_ci_high": 0.64,
-                "f1_micro": 0.5882352941176471,
-                "f1_micro_ci_low": 0.4444444444444444,
-                "f1_micro_ci_high": 0.7143825412589823
+                "accuracy": 0.52,
+                "accuracy_ci_low": 0.38,
+                "accuracy_ci_high": 0.66,
+                "f1_micro": 0.5909090909090909,
+                "f1_micro_ci_low": 0.449438202247191,
+                "f1_micro_ci_high": 0.723404255319149
             },
-            "score": 0.702719797596458,
+            "score": 0.7072192513368984,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.12,
-                "score": 0.12,
-                "score_name": "program_accuracy",
                 "execution_accuracy": 0.11,
+                "program_accuracy": 0.13,
+                "score": 0.13,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.06,
+                "execution_accuracy_ci_high": 0.18,
                 "program_accuracy_ci_low": 0.07,
-                "program_accuracy_ci_high": 0.19,
+                "program_accuracy_ci_high": 0.2,
                 "score_ci_low": 0.07,
-                "score_ci_high": 0.19,
-                "execution_accuracy_ci_low": 0.05,
-                "execution_accuracy_ci_high": 0.18
+                "score_ci_high": 0.2
             },
-            "score": 0.12,
+            "score": 0.13,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.4313706117093708,
-                "recall": 0.6100473634574232,
-                "f1": 0.46154026066287224,
-                "precision_ci_low": 0.39954934305142603,
-                "precision_ci_high": 0.4688092947200737,
-                "recall_ci_low": 0.561785694956309,
-                "recall_ci_high": 0.6478852680062104,
-                "f1_ci_low": 0.4348230402373462,
-                "f1_ci_high": 0.4887575867947463,
+                "precision": 0.4404755567778122,
+                "recall": 0.5911143426569531,
+                "f1": 0.46343593017736845,
+                "precision_ci_low": 0.40942454427118513,
+                "precision_ci_high": 0.4756683543235914,
+                "recall_ci_low": 0.5490257069385951,
+                "recall_ci_high": 0.6315097910079295,
+                "f1_ci_low": 0.43529875211270735,
+                "f1_ci_high": 0.48995060900564,
                 "score_name": "f1",
-                "score": 0.46154026066287224,
-                "score_ci_high": 0.4887575867947463,
-                "score_ci_low": 0.4348230402373462,
+                "score": 0.46343593017736845,
+                "score_ci_high": 0.48995060900564,
+                "score_ci_low": 0.43529875211270735,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6853316521644592,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7231194680929184,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6613810208439826,
-                "faithfullness_f1_token_overlap": 0.42865612410785836,
-                "faithfullness_recall_token_overlap": 0.3474276364400726,
-                "faithfullness_precision_token_overlap": 0.6951614384100505,
-                "correctness_f1_token_overlap": 0.46154026066287224,
-                "correctness_recall_token_overlap": 0.6100473634574232,
-                "correctness_precision_token_overlap": 0.4313706117093708
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6859477424621582,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7173790216445923,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.667030681669712,
+                "faithfullness_f1_token_overlap": 0.42049921925026323,
+                "faithfullness_recall_token_overlap": 0.32963765058230776,
+                "faithfullness_precision_token_overlap": 0.7030138840298064,
+                "correctness_f1_token_overlap": 0.46343593017736845,
+                "correctness_recall_token_overlap": 0.5911143426569531,
+                "correctness_precision_token_overlap": 0.4404755567778122
             },
-            "score": 0.46154026066287224,
+            "score": 0.46343593017736845,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -722,85 +722,85 @@
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.74,
-                "accuracy_ci_low": 0.65,
-                "accuracy_ci_high": 0.83,
+                "accuracy": 0.76,
+                "accuracy_ci_low": 0.67,
+                "accuracy_ci_high": 0.8367263963619302,
                 "score_name": "accuracy",
-                "score": 0.74,
-                "score_ci_high": 0.83,
-                "score_ci_low": 0.65,
+                "score": 0.76,
+                "score_ci_high": 0.8367263963619302,
+                "score_ci_low": 0.67,
                 "num_of_instances": 100
             },
-            "score": 0.575,
+            "score": 0.585,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 0.0,
-                "severity_high": 0.0,
-                "severity_medium": 1.0,
-                "severity_low": 99.0,
-                "category_harmful_info": 0.8524156869463032,
-                "category_deception": 0.8773881010334903,
-                "category_explicit_content": 0.859378189149529,
-                "category_violence": 0.8398635849356652,
-                "category_discrimination": 0.8441331806400465,
-                "category_substance_abuse": 0.8779810434931683,
-                "category_pii": 0.8530489449147826,
-                "safety": 0.852252782159485,
-                "safety_ci_low": 0.832847680014418,
-                "safety_ci_high": 0.8708004382240435,
+                "severity_critical": 1.0,
+                "severity_high": 1.0,
+                "severity_medium": 0.0,
+                "severity_low": 98.0,
+                "category_harmful_info": 0.8527039374550066,
+                "category_deception": 0.8602375103668735,
+                "category_explicit_content": 0.803066421337802,
+                "category_violence": 0.8417096956973038,
+                "category_discrimination": 0.8946837603099762,
+                "category_substance_abuse": 0.9066174957487318,
+                "category_pii": 0.8156786152610073,
+                "safety": 0.8487480249359376,
+                "safety_ci_low": 0.8219473814824015,
+                "safety_ci_high": 0.8711516865247095,
                 "score_name": "safety",
-                "score": 0.852252782159485,
-                "score_ci_high": 0.8708004382240435,
-                "score_ci_low": 0.832847680014418,
+                "score": 0.8487480249359376,
+                "score_ci_high": 0.8711516865247095,
+                "score_ci_low": 0.8219473814824015,
                 "num_of_instances": 100
             },
-            "score": 0.852252782159485,
+            "score": 0.8487480249359376,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge2": 0.17642133353378867,
-                "rougeL": 0.2631037191491013,
-                "score": 0.2631037191491013,
+                "rougeL": 0.2727278562571478,
+                "score": 0.2727278562571478,
                 "score_name": "rougeL",
-                "rouge1": 0.4091466523448068,
-                "rougeLsum": 0.34021391474003543,
-                "rouge2_ci_low": 0.16326857616870982,
-                "rouge2_ci_high": 0.19157111379873681,
-                "rougeL_ci_low": 0.2489620557680663,
-                "rougeL_ci_high": 0.27896256764413024,
-                "score_ci_low": 0.2489620557680663,
-                "score_ci_high": 0.27896256764413024,
-                "rouge1_ci_low": 0.3876453159453502,
-                "rouge1_ci_high": 0.4301714881055239,
-                "rougeLsum_ci_low": 0.31999068104597417,
-                "rougeLsum_ci_high": 0.35802162862605086
+                "rougeLsum": 0.3519467090096163,
+                "rouge2": 0.18296572636942984,
+                "rouge1": 0.4144140933241796,
+                "rougeL_ci_low": 0.2572827822057084,
+                "rougeL_ci_high": 0.2883182453977421,
+                "score_ci_low": 0.2572827822057084,
+                "score_ci_high": 0.2883182453977421,
+                "rougeLsum_ci_low": 0.33038241359811793,
+                "rougeLsum_ci_high": 0.36903407656270554,
+                "rouge2_ci_low": 0.17027692857079424,
+                "rouge2_ci_high": 0.195049523425454,
+                "rouge1_ci_low": 0.3899436362671578,
+                "rouge1_ci_high": 0.43263656274209517
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge2": 0.012194115618038825,
-                "rougeL": 0.07205757337255324,
-                "score": 0.07205757337255324,
+                "rougeL": 0.07189383629877291,
+                "score": 0.07189383629877291,
                 "score_name": "rougeL",
-                "rouge1": 0.0979610101102933,
-                "rougeLsum": 0.08211458868414816,
-                "rouge2_ci_low": 0.008497690786302959,
-                "rouge2_ci_high": 0.017709346128850476,
-                "rougeL_ci_low": 0.06301430117089502,
-                "rougeL_ci_high": 0.08066813507295256,
-                "score_ci_low": 0.06301430117089502,
-                "score_ci_high": 0.08066813507295256,
-                "rouge1_ci_low": 0.08482616596430592,
-                "rouge1_ci_high": 0.11192239519184158,
-                "rougeLsum_ci_low": 0.0716126906179182,
-                "rougeLsum_ci_high": 0.09246495633996564
+                "rougeLsum": 0.07911875003330153,
+                "rouge2": 0.010292492958310924,
+                "rouge1": 0.09538304929678695,
+                "rougeL_ci_low": 0.062145811602521625,
+                "rougeL_ci_high": 0.0812364852591871,
+                "score_ci_low": 0.062145811602521625,
+                "score_ci_high": 0.0812364852591871,
+                "rougeLsum_ci_low": 0.06822735527434497,
+                "rougeLsum_ci_high": 0.08982335497305671,
+                "rouge2_ci_low": 0.006919922257555467,
+                "rouge2_ci_high": 0.014744400890407348,
+                "rouge1_ci_low": 0.08225253893978755,
+                "rouge1_ci_high": 0.10869171536999694
             },
-            "score": 0.16758064626082728,
+            "score": 0.17231084627796034,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    117,
-                    57,
-                    33,
-                    20
+                    119,
+                    62,
+                    35,
+                    21
                 ],
                 "totals": [
-                    201,
-                    195,
-                    189,
-                    183
+                    230,
+                    224,
+                    218,
+                    212
                 ],
                 "precisions": [
-                    0.582089552238806,
-                    0.2923076923076923,
-                    0.1746031746031746,
-                    0.1092896174863388
+                    0.5173913043478261,
+                    0.27678571428571425,
+                    0.1605504587155963,
+                    0.0990566037735849
                 ],
-                "bp": 0.9657735711441044,
-                "sys_len": 201,
+                "bp": 1.0,
+                "sys_len": 230,
                 "ref_len": 208,
-                "sacrebleu": 0.23053697440015153,
-                "score": 0.23053697440015153,
+                "sacrebleu": 0.21845623538755882,
+                "score": 0.21845623538755882,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10517490265827865,
-                "score_ci_high": 0.3940190928317325,
-                "sacrebleu_ci_low": 0.10517490265827865,
-                "sacrebleu_ci_high": 0.3940190928317325
+                "score_ci_low": 0.09083341323391328,
+                "score_ci_high": 0.3838181728128643,
+                "sacrebleu_ci_low": 0.09083341323391328,
+                "sacrebleu_ci_high": 0.3838181728128643
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    117,
-                    65,
-                    35,
-                    23
+                    118,
+                    61,
+                    34,
+                    22
                 ],
                 "totals": [
-                    213,
-                    207,
-                    201,
-                    195
+                    209,
+                    203,
+                    197,
+                    191
                 ],
                 "precisions": [
-                    0.5492957746478874,
-                    0.3140096618357488,
-                    0.17412935323383086,
-                    0.11794871794871796
+                    0.5645933014354066,
+                    0.30049261083743845,
+                    0.17258883248730963,
+                    0.11518324607329843
                 ],
                 "bp": 1.0,
-                "sys_len": 213,
+                "sys_len": 209,
                 "ref_len": 208,
-                "sacrebleu": 0.2439656149705105,
-                "score": 0.2439656149705105,
+                "sacrebleu": 0.2409865923854554,
+                "score": 0.2409865923854554,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.14790087731817905,
-                "score_ci_high": 0.33202464586887365,
-                "sacrebleu_ci_low": 0.14790087731817905,
-                "sacrebleu_ci_high": 0.33202464586887365
+                "score_ci_low": 0.14821008407472766,
+                "score_ci_high": 0.3454690019726006,
+                "sacrebleu_ci_low": 0.14821008407472766,
+                "sacrebleu_ci_high": 0.3454690019726006
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    72,
-                    23,
+                    81,
+                    26,
                     11,
                     6
                 ],
                 "totals": [
-                    206,
-                    200,
-                    194,
-                    188
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.34951456310679613,
-                    0.115,
-                    0.05670103092783505,
-                    0.031914893617021274
+                    0.37674418604651166,
+                    0.12440191387559808,
+                    0.054187192118226604,
+                    0.03045685279187817
                 ],
-                "bp": 0.9855424223451845,
-                "sys_len": 206,
+                "bp": 1.0,
+                "sys_len": 215,
                 "ref_len": 209,
-                "sacrebleu": 0.09101483608425708,
-                "score": 0.09101483608425708,
+                "sacrebleu": 0.09378077622334098,
+                "score": 0.09378077622334098,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.04163319689445597,
-                "score_ci_high": 0.12744602542291247,
-                "sacrebleu_ci_low": 0.04163319689445597,
-                "sacrebleu_ci_high": 0.12744602542291247
+                "score_ci_low": 0.023731214333943484,
+                "score_ci_high": 0.15084948419488436,
+                "sacrebleu_ci_low": 0.023731214333943484,
+                "sacrebleu_ci_high": 0.15084948419488436
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    113,
-                    54,
-                    29,
-                    13
+                    121,
+                    63,
+                    34,
+                    18
                 ],
                 "totals": [
-                    224,
-                    218,
-                    212,
-                    206
+                    211,
+                    205,
+                    199,
+                    193
                 ],
                 "precisions": [
-                    0.5044642857142857,
-                    0.24770642201834864,
-                    0.13679245283018868,
-                    0.06310679611650485
+                    0.5734597156398105,
+                    0.3073170731707317,
+                    0.1708542713567839,
+                    0.09326424870466321
                 ],
-                "bp": 1.0,
-                "sys_len": 224,
+                "bp": 0.9765818792478103,
+                "sys_len": 211,
                 "ref_len": 216,
-                "sacrebleu": 0.1812284975765625,
-                "score": 0.1812284975765625,
+                "sacrebleu": 0.22481036027314003,
+                "score": 0.22481036027314003,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10521667883417998,
-                "score_ci_high": 0.29917894657147576,
-                "sacrebleu_ci_low": 0.10521667883417998,
-                "sacrebleu_ci_high": 0.29917894657147576
+                "score_ci_low": 0.143685448931687,
+                "score_ci_high": 0.3383200604081932,
+                "sacrebleu_ci_low": 0.143685448931687,
+                "sacrebleu_ci_high": 0.3383200604081932
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    162,
-                    103,
-                    68,
-                    48
+                    167,
+                    112,
+                    77,
+                    55
                 ],
                 "totals": [
-                    246,
-                    240,
-                    234,
-                    228
+                    238,
+                    232,
+                    226,
+                    220
                 ],
                 "precisions": [
-                    0.6585365853658537,
-                    0.42916666666666664,
-                    0.2905982905982906,
-                    0.2105263157894737
+                    0.7016806722689075,
+                    0.48275862068965514,
+                    0.34070796460176994,
+                    0.25
                 ],
                 "bp": 1.0,
-                "sys_len": 246,
+                "sys_len": 238,
                 "ref_len": 235,
-                "sacrebleu": 0.36261964969994975,
-                "score": 0.36261964969994975,
+                "sacrebleu": 0.41214303191378043,
+                "score": 0.41214303191378043,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.29613813398752276,
-                "score_ci_high": 0.4664466752414592,
-                "sacrebleu_ci_low": 0.29613813398752276,
-                "sacrebleu_ci_high": 0.4664466752414592
+                "score_ci_low": 0.3395229044712486,
+                "score_ci_high": 0.4807451635871571,
+                "sacrebleu_ci_low": 0.3395229044712486,
+                "sacrebleu_ci_high": 0.4807451635871571
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    102,
-                    39,
-                    22,
-                    11
+                    111,
+                    40,
+                    21,
+                    12
                 ],
                 "totals": [
-                    348,
-                    342,
-                    336,
-                    330
+                    346,
+                    340,
+                    334,
+                    328
                 ],
                 "precisions": [
-                    0.29310344827586204,
-                    0.11403508771929825,
-                    0.06547619047619048,
-                    0.03333333333333333
+                    0.32080924855491333,
+                    0.11764705882352942,
+                    0.06287425149700598,
+                    0.03658536585365854
                 ],
                 "bp": 1.0,
-                "sys_len": 348,
+                "sys_len": 346,
                 "ref_len": 249,
-                "sacrebleu": 0.09241775072026762,
-                "score": 0.09241775072026762,
+                "sacrebleu": 0.09652771953936146,
+                "score": 0.09652771953936146,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.03911145943513657,
-                "score_ci_high": 0.13521883464235146,
-                "sacrebleu_ci_low": 0.03911145943513657,
-                "sacrebleu_ci_high": 0.13521883464235146
+                "score_ci_low": 0.039423637442761,
+                "score_ci_high": 0.16445838229165144,
+                "sacrebleu_ci_low": 0.039423637442761,
+                "sacrebleu_ci_high": 0.16445838229165144
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
                     149,
-                    95,
+                    99,
                     66,
                     46
                 ],
                 "totals": [
-                    230,
-                    224,
-                    218,
-                    212
+                    229,
+                    223,
+                    217,
+                    211
                 ],
                 "precisions": [
-                    0.6478260869565218,
-                    0.42410714285714285,
-                    0.3027522935779816,
-                    0.2169811320754717
+                    0.6506550218340611,
+                    0.4439461883408072,
+                    0.30414746543778803,
+                    0.21800947867298578
                 ],
                 "bp": 1.0,
-                "sys_len": 230,
+                "sys_len": 229,
                 "ref_len": 222,
-                "sacrebleu": 0.3665311234426107,
-                "score": 0.3665311234426107,
+                "sacrebleu": 0.37201476231316355,
+                "score": 0.37201476231316355,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2708503915486859,
-                "score_ci_high": 0.4439101310395995,
-                "sacrebleu_ci_low": 0.2708503915486859,
-                "sacrebleu_ci_high": 0.4439101310395995
+                "score_ci_low": 0.2874102896870317,
+                "score_ci_high": 0.4690981288042721,
+                "sacrebleu_ci_low": 0.2874102896870317,
+                "sacrebleu_ci_high": 0.4690981288042721
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    110,
-                    42,
-                    22,
-                    15
+                    103,
+                    41,
+                    28,
+                    20
                 ],
                 "totals": [
-                    240,
-                    234,
-                    228,
-                    222
+                    271,
+                    265,
+                    259,
+                    253
                 ],
                 "precisions": [
-                    0.45833333333333337,
-                    0.1794871794871795,
-                    0.09649122807017545,
-                    0.06756756756756757
+                    0.3800738007380074,
+                    0.15471698113207547,
+                    0.1081081081081081,
+                    0.07905138339920949
                 ],
                 "bp": 1.0,
-                "sys_len": 240,
+                "sys_len": 271,
                 "ref_len": 230,
-                "sacrebleu": 0.1521809352229689,
-                "score": 0.1521809352229689,
+                "sacrebleu": 0.14972468725087626,
+                "score": 0.14972468725087626,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.04331464284837397,
-                "score_ci_high": 0.28507289956238885,
-                "sacrebleu_ci_low": 0.04331464284837397,
-                "sacrebleu_ci_high": 0.28507289956238885
+                "score_ci_low": 0.09950692459247845,
+                "score_ci_high": 0.2774830219069996,
+                "sacrebleu_ci_low": 0.09950692459247845,
+                "sacrebleu_ci_high": 0.2774830219069996
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    153,
-                    89,
-                    58,
-                    39
+                    146,
+                    82,
+                    50,
+                    34
                 ],
                 "totals": [
-                    235,
-                    229,
-                    223,
-                    217
+                    232,
+                    226,
+                    220,
+                    214
                 ],
                 "precisions": [
-                    0.651063829787234,
-                    0.38864628820960695,
-                    0.2600896860986547,
-                    0.17972350230414746
+                    0.6293103448275862,
+                    0.36283185840707965,
+                    0.22727272727272727,
+                    0.15887850467289721
                 ],
-                "bp": 0.9665303748102905,
-                "sys_len": 235,
+                "bp": 0.9536926844755759,
+                "sys_len": 232,
                 "ref_len": 243,
-                "sacrebleu": 0.3187440093392008,
-                "score": 0.3187440093392008,
+                "sacrebleu": 0.2873784110095771,
+                "score": 0.2873784110095771,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23704783766140267,
-                "score_ci_high": 0.35626973427173086,
-                "sacrebleu_ci_low": 0.23704783766140267,
-                "sacrebleu_ci_high": 0.35626973427173086
+                "score_ci_low": 0.215659644222719,
+                "score_ci_high": 0.33135108326345925,
+                "sacrebleu_ci_low": 0.215659644222719,
+                "sacrebleu_ci_high": 0.33135108326345925
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    140,
-                    88,
-                    53,
-                    32
+                    137,
+                    84,
+                    56,
+                    38
                 ],
                 "totals": [
-                    213,
-                    207,
-                    201,
-                    195
+                    220,
+                    214,
+                    208,
+                    202
                 ],
                 "precisions": [
-                    0.6572769953051644,
-                    0.42512077294685985,
-                    0.263681592039801,
-                    0.16410256410256407
+                    0.6227272727272727,
+                    0.3925233644859813,
+                    0.2692307692307692,
+                    0.18811881188118812
                 ],
                 "bp": 1.0,
-                "sys_len": 213,
+                "sys_len": 220,
                 "ref_len": 208,
-                "sacrebleu": 0.3315995897054171,
-                "score": 0.3315995897054171,
+                "sacrebleu": 0.33356469620008616,
+                "score": 0.33356469620008616,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3023185103522073,
-                "score_ci_high": 0.36769424496487263,
-                "sacrebleu_ci_low": 0.3023185103522073,
-                "sacrebleu_ci_high": 0.36769424496487263
+                "score_ci_low": 0.24321947226818338,
+                "score_ci_high": 0.448027949444875,
+                "sacrebleu_ci_low": 0.24321947226818338,
+                "sacrebleu_ci_high": 0.448027949444875
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    112,
-                    56,
+                    114,
+                    53,
                     31,
-                    18
+                    20
                 ],
                 "totals": [
-                    262,
-                    256,
-                    250,
-                    244
+                    223,
+                    217,
+                    211,
+                    205
                 ],
                 "precisions": [
-                    0.42748091603053434,
-                    0.21875,
-                    0.124,
-                    0.07377049180327869
+                    0.5112107623318386,
+                    0.24423963133640553,
+                    0.14691943127962084,
+                    0.0975609756097561
                 ],
                 "bp": 1.0,
-                "sys_len": 262,
+                "sys_len": 223,
                 "ref_len": 208,
-                "sacrebleu": 0.17101818352211745,
-                "score": 0.17101818352211745,
+                "sacrebleu": 0.20568038392617954,
+                "score": 0.20568038392617954,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11747970698212559,
-                "score_ci_high": 0.25765320515004,
-                "sacrebleu_ci_low": 0.11747970698212559,
-                "sacrebleu_ci_high": 0.25765320515004
+                "score_ci_low": 0.10798401541766527,
+                "score_ci_high": 0.27947103589520705,
+                "sacrebleu_ci_low": 0.10798401541766527,
+                "sacrebleu_ci_high": 0.27947103589520705
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    108,
-                    47,
-                    25,
-                    13
+                    96,
+                    35,
+                    12,
+                    4
                 ],
                 "totals": [
-                    217,
-                    211,
-                    205,
-                    199
+                    206,
+                    200,
+                    194,
+                    188
                 ],
                 "precisions": [
-                    0.4976958525345622,
-                    0.22274881516587677,
-                    0.12195121951219512,
-                    0.06532663316582915
+                    0.46601941747572817,
+                    0.175,
+                    0.061855670103092786,
+                    0.02127659574468085
                 ],
-                "bp": 1.0,
-                "sys_len": 217,
+                "bp": 0.9903382397772544,
+                "sys_len": 206,
                 "ref_len": 208,
-                "sacrebleu": 0.1723907511301038,
-                "score": 0.1723907511301038,
+                "sacrebleu": 0.1008009159878086,
+                "score": 0.1008009159878086,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12055708968153714,
-                "score_ci_high": 0.23536159723298522,
-                "sacrebleu_ci_low": 0.12055708968153714,
-                "sacrebleu_ci_high": 0.23536159723298522
+                "score_ci_low": 0.04782093685517172,
+                "score_ci_high": 0.15317427576781073,
+                "sacrebleu_ci_low": 0.04782093685517172,
+                "sacrebleu_ci_high": 0.15317427576781073
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    138,
-                    90,
-                    56,
-                    42
+                    126,
+                    72,
+                    41,
+                    27
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    219,
+                    213,
+                    207,
+                    201
                 ],
                 "precisions": [
-                    0.6540284360189574,
-                    0.4390243902439025,
-                    0.2814070351758794,
-                    0.21761658031088082
+                    0.5753424657534246,
+                    0.3380281690140845,
+                    0.19806763285024154,
+                    0.13432835820895522
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 219,
                 "ref_len": 208,
-                "sacrebleu": 0.3641481038335531,
-                "score": 0.3641481038335531,
+                "sacrebleu": 0.2682039287808841,
+                "score": 0.2682039287808841,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2224318478297402,
-                "score_ci_high": 0.45657021838402995,
-                "sacrebleu_ci_low": 0.2224318478297402,
-                "sacrebleu_ci_high": 0.45657021838402995
+                "score_ci_low": 0.16947811739242713,
+                "score_ci_high": 0.39632641564304394,
+                "sacrebleu_ci_low": 0.16947811739242713,
+                "sacrebleu_ci_high": 0.39632641564304394
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    136,
-                    81,
-                    57,
-                    41
+                    131,
+                    75,
+                    50,
+                    34
                 ],
                 "totals": [
+                    235,
                     229,
                     223,
-                    217,
-                    211
+                    217
                 ],
                 "precisions": [
-                    0.5938864628820961,
-                    0.36322869955156956,
-                    0.2626728110599078,
-                    0.19431279620853079
+                    0.5574468085106382,
+                    0.32751091703056767,
+                    0.22421524663677128,
+                    0.1566820276497696
                 ],
                 "bp": 1.0,
-                "sys_len": 229,
+                "sys_len": 235,
                 "ref_len": 208,
-                "sacrebleu": 0.3239291461681843,
-                "score": 0.3239291461681843,
+                "sacrebleu": 0.28299475389639145,
+                "score": 0.28299475389639145,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23531941746553897,
-                "score_ci_high": 0.4842839193385085,
-                "sacrebleu_ci_low": 0.23531941746553897,
-                "sacrebleu_ci_high": 0.4842839193385085
+                "score_ci_low": 0.16939445196325967,
+                "score_ci_high": 0.37514174425200963,
+                "sacrebleu_ci_low": 0.16939445196325967,
+                "sacrebleu_ci_high": 0.37514174425200963
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    128,
-                    64,
-                    35,
+                    127,
+                    68,
+                    36,
                     23
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    223,
+                    217,
+                    211,
+                    205
                 ],
                 "precisions": [
-                    0.5688888888888889,
-                    0.2922374429223744,
-                    0.1643192488262911,
-                    0.1111111111111111
+                    0.5695067264573991,
+                    0.3133640552995392,
+                    0.17061611374407584,
+                    0.1121951219512195
                 ],
                 "bp": 1.0,
-                "sys_len": 225,
+                "sys_len": 223,
                 "ref_len": 208,
-                "sacrebleu": 0.23472119660866142,
-                "score": 0.23472119660866142,
+                "sacrebleu": 0.24176059414017798,
+                "score": 0.24176059414017798,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11338576888580632,
-                "score_ci_high": 0.32460680792548485,
-                "sacrebleu_ci_low": 0.11338576888580632,
-                "sacrebleu_ci_high": 0.32460680792548485
+                "score_ci_low": 0.1347717277253673,
+                "score_ci_high": 0.31632416882417563,
+                "sacrebleu_ci_low": 0.1347717277253673,
+                "sacrebleu_ci_high": 0.31632416882417563
             },
-            "score": 0.24246975749496777,
+            "score": 0.2352551899485188,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.4017742565069269,
+        "score": 0.4098791026510746,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T15-15-09_evaluation_results.json → 2025-07-03T07-36-22_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T19:15:05.019850Z",
+        "timestamp_utc": "2025-07-03T11:36:19.083305Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,23 +176,23 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.6666666666666666,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.1111111111111111,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.6666666666666666,
-                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
                 "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.6666666666666666,
+                "score": 0.5555555555555556,
                 "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.3333333333333333,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_gender_identity": {
@@ -206,13 +206,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_nationality": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
             "safety_bbq_physical_appearance": {
@@ -227,12 +227,12 @@
             },
             "safety_bbq_race_ethnicity": {
                 "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_low": 0.47716657027690984,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
                 "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.4444444444444444,
+                "score_ci_low": 0.47716657027690984,
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_gender": {
@@ -246,13 +246,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.3333333333333333,
+                "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
             "safety_bbq_religion": {
@@ -276,63 +276,63 @@
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.47716657027690984,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.3333333333333333,
+                "score_ci_low": 0.47716657027690984,
                 "num_of_instances": 9
             },
-            "score": 0.7474747474747474,
+            "score": 0.7676767676767676,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.6125,
-                "score": 0.6125,
+                "llama_3_70b_instruct_template_arena_hard": 0.6078431372549019,
+                "score": 0.6078431372549019,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.6125,
+            "score": 0.6078431372549019,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.37500000000000006,
-                "f1_Organization": 0.2857142857142857,
-                "f1_Location": 0.13793103448275862,
-                "f1_macro": 0.26621510673234816,
-                "recall_macro": 0.18616287094547965,
-                "precision_macro": 0.4984126984126984,
-                "in_classes_support": 0.509090909090909,
-                "f1_micro": 0.2153846153846154,
-                "recall_micro": 0.18666666666666668,
-                "precision_micro": 0.2545454545454545,
-                "score": 0.2153846153846154,
+                "f1_Person": 0.6842105263157895,
+                "f1_Organization": 0.4799999999999999,
+                "f1_Location": 0.26666666666666666,
+                "f1_macro": 0.4769590643274853,
+                "recall_macro": 0.38681849551414765,
+                "precision_macro": 0.6929292929292928,
+                "in_classes_support": 0.7678571428571428,
+                "f1_micro": 0.44274809160305345,
+                "recall_micro": 0.38666666666666666,
+                "precision_micro": 0.5178571428571429,
+                "score": 0.44274809160305345,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.11288016143257878,
-                "score_ci_high": 0.3309060924550398,
-                "f1_micro_ci_low": 0.11288016143257878,
-                "f1_micro_ci_high": 0.3309060924550398
+                "score_ci_low": 0.3289856810851347,
+                "score_ci_high": 0.5750204568214298,
+                "f1_micro_ci_low": 0.3289856810851347,
+                "f1_micro_ci_high": 0.5750204568214298
             },
-            "score": 0.2153846153846154,
+            "score": 0.44274809160305345,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
@@ -346,31 +346,31 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.0,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.0,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.0,
+                "score_ci_high": 0.0,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
@@ -396,12 +396,12 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
-                "accuracy": 0.2857142857142857,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -416,13 +416,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.6807203593841678,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.6807203593841678,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
@@ -436,22 +436,22 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.0,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -471,91 +471,91 @@
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.2723076923076923,
+                "f1_macro": 0.3290909090909091,
                 "f1_suggestive": 0.5,
-                "f1_arbitrary": 0.46153846153846156,
+                "f1_arbitrary": 0.5454545454545454,
                 "f1_generic": 0.0,
                 "f1_fanciful": 0.0,
-                "f1_descriptive": 0.4,
-                "f1_macro_ci_low": 0.13333333333333336,
-                "f1_macro_ci_high": 0.4398453947518335,
+                "f1_descriptive": 0.6,
+                "f1_macro_ci_low": 0.1757208732808955,
+                "f1_macro_ci_high": 0.4941262683635378,
                 "score_name": "f1_micro",
-                "score": 0.3684210526315789,
-                "score_ci_high": 0.5809586841346625,
-                "score_ci_low": 0.16216216216216217,
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.22857142857142856,
                 "num_of_instances": 20,
-                "accuracy": 0.35,
-                "accuracy_ci_low": 0.15,
-                "accuracy_ci_high": 0.55,
-                "f1_micro": 0.3684210526315789,
-                "f1_micro_ci_low": 0.16216216216216217,
-                "f1_micro_ci_high": 0.5809586841346625
+                "accuracy": 0.4,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.4444444444444444,
+                "f1_micro_ci_low": 0.22857142857142856,
+                "f1_micro_ci_high": 0.6666666666666666
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.6222222222222222,
+                "f1_macro": 0.6000000000000001,
                 "f1_no": 0.8,
-                "f1_yes": 0.4444444444444444,
+                "f1_yes": 0.4,
                 "f1_macro_ci_low": 0.3939393939393939,
-                "f1_macro_ci_high": 0.9235569748599021,
+                "f1_macro_ci_high": 0.9134199134199135,
                 "score_name": "f1_micro",
-                "score": 0.717948717948718,
+                "score": 0.7,
                 "score_ci_high": 0.9,
-                "score_ci_low": 0.47368421052631576,
+                "score_ci_low": 0.4568473225601714,
                 "num_of_instances": 20,
                 "accuracy": 0.7,
-                "accuracy_ci_low": 0.45,
-                "accuracy_ci_high": 0.85,
-                "f1_micro": 0.717948717948718,
-                "f1_micro_ci_low": 0.47368421052631576,
+                "accuracy_ci_low": 0.46284048542422074,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7,
+                "f1_micro_ci_low": 0.4568473225601714,
                 "f1_micro_ci_high": 0.9
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.2119047619047619,
+                "f1_macro": 0.32142857142857145,
                 "f1_conclusion": 0.3333333333333333,
+                "f1_decree": 0.5,
                 "f1_issue": 0.25,
-                "f1_decree": 0.0,
-                "f1_rule": 0.0,
-                "f1_analysis": 0.5,
-                "f1_facts": 0.4,
+                "f1_rule": 0.5,
+                "f1_analysis": 0.6666666666666666,
+                "f1_facts": 0.0,
                 "f1_procedural history": 0.0,
-                "f1_macro_ci_low": 0.05607086200596039,
-                "f1_macro_ci_high": 0.466763791166963,
+                "f1_macro_ci_low": 0.13841395740078669,
+                "f1_macro_ci_high": 0.5943854196450136,
                 "score_name": "f1_micro",
-                "score": 0.2222222222222222,
-                "score_ci_high": 0.4666666666666667,
-                "score_ci_low": 0.05555555555555555,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.5176853452302272,
+                "score_ci_low": 0.06451612903225806,
                 "num_of_instances": 20,
-                "accuracy": 0.2,
-                "accuracy_ci_low": 0.05,
-                "accuracy_ci_high": 0.45,
-                "f1_micro": 0.2222222222222222,
-                "f1_micro_ci_low": 0.05555555555555555,
-                "f1_micro_ci_high": 0.4666666666666667
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.2857142857142857,
+                "f1_micro_ci_low": 0.06451612903225806,
+                "f1_micro_ci_high": 0.5176853452302272
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.43452380952380953,
-                "f1_yes": 0.5833333333333334,
-                "f1_no": 0.2857142857142857,
-                "f1_macro_ci_low": 0.25407682715906454,
-                "f1_macro_ci_high": 0.677425770108348,
+                "f1_macro": 0.5478260869565217,
+                "f1_yes": 0.6956521739130435,
+                "f1_no": 0.4,
+                "f1_macro_ci_low": 0.33939393939393936,
+                "f1_macro_ci_high": 0.7815126050420168,
                 "score_name": "f1_micro",
-                "score": 0.47368421052631576,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.2564102564102564,
+                "score": 0.5789473684210527,
+                "score_ci_high": 0.7692307692307693,
+                "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 20,
-                "accuracy": 0.45,
-                "accuracy_ci_low": 0.25,
-                "accuracy_ci_high": 0.65,
-                "f1_micro": 0.47368421052631576,
-                "f1_micro_ci_low": 0.2564102564102564,
-                "f1_micro_ci_high": 0.6666666666666666
+                "accuracy": 0.55,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.75,
+                "f1_micro": 0.5789473684210527,
+                "f1_micro_ci_low": 0.3333333333333333,
+                "f1_micro_ci_high": 0.7692307692307693
             },
             "legalbench_proa": {
-                "f1_macro": 0.788888888888889,
-                "f1_yes": 0.8,
-                "f1_no": 0.7777777777777778,
-                "f1_macro_ci_low": 0.5831476917982417,
-                "f1_macro_ci_high": 0.9136904761904762,
+                "f1_macro": 0.7781954887218046,
+                "f1_yes": 0.7142857142857143,
+                "f1_no": 0.8421052631578947,
+                "f1_macro_ci_low": 0.5689078197436116,
+                "f1_macro_ci_high": 0.9097245188862976,
                 "score_name": "f1_micro",
                 "score": 0.7878787878787878,
                 "score_ci_high": 0.8888888888888888,
@@ -568,170 +568,170 @@
                 "f1_micro_ci_low": 0.5714285714285714,
                 "f1_micro_ci_high": 0.8888888888888888
             },
-            "score": 0.5140309982415245,
+            "score": 0.5593969772917141,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.3572280497280497,
+                "f1_macro": 0.3952113526570048,
                 "f1_cars": 0.6,
-                "f1_pc hardware": 0.38095238095238093,
+                "f1_pc hardware": 0.43478260869565216,
                 "f1_windows x": 0.0,
-                "f1_computer graphics": 0.5,
-                "f1_atheism": 0.0,
+                "f1_atheism": 0.25,
                 "f1_religion": 0.0,
                 "f1_medicine": 0.8571428571428571,
                 "f1_christianity": 0.0,
+                "f1_computer graphics": 0.5,
                 "f1_microsoft windows": 0.8,
                 "f1_middle east": 0.25,
-                "f1_politics": 0.3076923076923077,
+                "f1_politics": 0.375,
                 "f1_motorcycles": 0.4444444444444444,
-                "f1_mac hardware": 0.3333333333333333,
-                "f1_for sale": 0.3333333333333333,
-                "f1_guns": 0.2857142857142857,
+                "f1_mac hardware": 0.0,
+                "f1_for sale": 0.5714285714285714,
+                "f1_guns": 0.5,
                 "f1_space": 0.5714285714285714,
                 "f1_cryptography": 0.0,
-                "f1_baseball": 0.9090909090909091,
-                "f1_hockey": 0.5714285714285714,
+                "f1_baseball": 1.0,
+                "f1_hockey": 0.75,
                 "f1_electronics": 0.0,
-                "f1_macro_ci_low": 0.28522044565980664,
-                "f1_macro_ci_high": 0.45653572695267247,
+                "f1_macro_ci_low": 0.3224857117174346,
+                "f1_macro_ci_high": 0.4919903471692233,
                 "score_name": "f1_micro",
-                "score": 0.4166666666666667,
-                "score_ci_high": 0.5174687718987632,
-                "score_ci_low": 0.3130413460163748,
+                "score": 0.4606741573033708,
+                "score_ci_high": 0.5673987245542301,
+                "score_ci_low": 0.3609984101265133,
                 "num_of_instances": 100,
-                "accuracy": 0.35,
-                "accuracy_ci_low": 0.26,
-                "accuracy_ci_high": 0.45,
-                "f1_micro": 0.4166666666666667,
-                "f1_micro_ci_low": 0.3130413460163748,
-                "f1_micro_ci_high": 0.5174687718987632
+                "accuracy": 0.41,
+                "accuracy_ci_low": 0.32,
+                "accuracy_ci_high": 0.52,
+                "f1_micro": 0.4606741573033708,
+                "f1_micro_ci_low": 0.3609984101265133,
+                "f1_micro_ci_high": 0.5673987245542301
             },
-            "score": 0.4166666666666667,
+            "score": 0.4606741573033708,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.7443746729461015,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8923076923076924,
-                "f1_money transfer or virtual currency or money service": 0.8,
+                "f1_macro": 0.6786958139899316,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9090909090909091,
+                "f1_money transfer or virtual currency or money service": 0.5,
                 "f1_mortgage": 0.6666666666666666,
-                "f1_credit card or prepaid card": 0.7619047619047619,
-                "f1_debt collection": 0.6666666666666666,
-                "f1_checking or savings account": 0.9230769230769231,
+                "f1_credit card or prepaid card": 0.7,
+                "f1_debt collection": 0.7058823529411765,
+                "f1_checking or savings account": 0.7692307692307693,
                 "f1_payday loan or title loan or personal loan": 0.5,
-                "f1_macro_ci_low": 0.5205817875108439,
-                "f1_macro_ci_high": 0.8473977257078351,
+                "f1_macro_ci_low": 0.5183018601419674,
+                "f1_macro_ci_high": 0.8385575362562627,
                 "score_name": "f1_micro",
-                "score": 0.845360824742268,
-                "score_ci_high": 0.9035532994923858,
-                "score_ci_low": 0.7626425416851077,
+                "score": 0.8393782383419689,
+                "score_ci_high": 0.8979591836734694,
+                "score_ci_low": 0.7543122896467558,
                 "num_of_instances": 100,
-                "accuracy": 0.82,
-                "accuracy_ci_low": 0.73,
-                "accuracy_ci_high": 0.89,
-                "f1_micro": 0.845360824742268,
-                "f1_micro_ci_low": 0.7626425416851077,
-                "f1_micro_ci_high": 0.9035532994923858
+                "accuracy": 0.81,
+                "accuracy_ci_low": 0.72,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.8393782383419689,
+                "f1_micro_ci_low": 0.7543122896467558,
+                "f1_micro_ci_high": 0.8979591836734694
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6763879395458343,
+                "f1_macro": 0.6211833932657731,
                 "f1_mortgages and loans": 0.631578947368421,
-                "f1_credit card": 0.5555555555555556,
-                "f1_debt collection": 0.8571428571428571,
+                "f1_credit card": 0.631578947368421,
+                "f1_debt collection": 0.631578947368421,
+                "f1_credit reporting": 0.782608695652174,
                 "f1_retail banking": 0.42857142857142855,
-                "f1_credit reporting": 0.9090909090909091,
-                "f1_macro_ci_low": 0.5567123133423233,
-                "f1_macro_ci_high": 0.8148932069233683,
+                "f1_macro_ci_low": 0.4842400654698589,
+                "f1_macro_ci_high": 0.7592332532338871,
                 "score_name": "f1_micro",
-                "score": 0.7021276595744681,
-                "score_ci_high": 0.8163265306122449,
-                "score_ci_low": 0.5625,
+                "score": 0.6382978723404256,
+                "score_ci_high": 0.7580700999704794,
+                "score_ci_low": 0.4946236559139785,
                 "num_of_instances": 50,
-                "accuracy": 0.66,
-                "accuracy_ci_low": 0.52,
-                "accuracy_ci_high": 0.78,
-                "f1_micro": 0.7021276595744681,
-                "f1_micro_ci_low": 0.5625,
-                "f1_micro_ci_high": 0.8163265306122449
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.46,
+                "accuracy_ci_high": 0.7296431071552615,
+                "f1_micro": 0.6382978723404256,
+                "f1_micro_ci_low": 0.4946236559139785,
+                "f1_micro_ci_high": 0.7580700999704794
             },
-            "score": 0.773744242158368,
+            "score": 0.7388380553411973,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "execution_accuracy": 0.1,
-                "program_accuracy": 0.12,
-                "score": 0.12,
+                "program_accuracy": 0.09,
+                "score": 0.09,
                 "score_name": "program_accuracy",
+                "execution_accuracy": 0.1,
+                "program_accuracy_ci_low": 0.04,
+                "program_accuracy_ci_high": 0.17,
+                "score_ci_low": 0.04,
+                "score_ci_high": 0.17,
                 "execution_accuracy_ci_low": 0.05,
-                "execution_accuracy_ci_high": 0.18,
-                "program_accuracy_ci_low": 0.07,
-                "program_accuracy_ci_high": 0.2,
-                "score_ci_low": 0.07,
-                "score_ci_high": 0.2
+                "execution_accuracy_ci_high": 0.17
             },
-            "score": 0.12,
+            "score": 0.09,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.497952765281177,
-                "recall": 0.6516388140313351,
-                "f1": 0.5191763100507532,
-                "precision_ci_low": 0.45838123107587214,
-                "precision_ci_high": 0.5423702973763924,
-                "recall_ci_low": 0.6111720765957214,
-                "recall_ci_high": 0.6938024168384508,
-                "f1_ci_low": 0.4872713168221134,
-                "f1_ci_high": 0.5540774303790778,
+                "precision": 0.47761082372578506,
+                "recall": 0.6519026742512972,
+                "f1": 0.519200175226266,
+                "precision_ci_low": 0.4406041953630249,
+                "precision_ci_high": 0.5185697572184587,
+                "recall_ci_low": 0.6137821254589361,
+                "recall_ci_high": 0.6857346944815508,
+                "f1_ci_low": 0.4868309702128611,
+                "f1_ci_high": 0.5539790603518593,
                 "score_name": "f1",
-                "score": 0.5191763100507532,
-                "score_ci_high": 0.5540774303790778,
-                "score_ci_low": 0.4872713168221134,
+                "score": 0.519200175226266,
+                "score_ci_high": 0.5539790603518593,
+                "score_ci_low": 0.4868309702128611,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.7120448821783065,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7458498811721802,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6911013838648796,
-                "faithfullness_f1_token_overlap": 0.4492844732681271,
-                "faithfullness_recall_token_overlap": 0.356905942911477,
-                "faithfullness_precision_token_overlap": 0.7449470313452086,
-                "correctness_f1_token_overlap": 0.5191763100507532,
-                "correctness_recall_token_overlap": 0.6516388140313351,
-                "correctness_precision_token_overlap": 0.497952765281177
+                "correctness_f1_bert_score.deberta_large_mnli": 0.7154018753767013,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7530737143754959,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6891439840197563,
+                "faithfullness_f1_token_overlap": 0.46756124806620003,
+                "faithfullness_recall_token_overlap": 0.3785499103669753,
+                "faithfullness_precision_token_overlap": 0.7384812140442224,
+                "correctness_f1_token_overlap": 0.519200175226266,
+                "correctness_recall_token_overlap": 0.6519026742512972,
+                "correctness_precision_token_overlap": 0.47761082372578506
             },
-            "score": 0.5191763100507532,
+            "score": 0.519200175226266,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.44,
-                "accuracy_ci_low": 0.34,
-                "accuracy_ci_high": 0.54,
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.35,
+                "accuracy_ci_high": 0.55,
                 "score_name": "accuracy",
-                "score": 0.44,
-                "score_ci_high": 0.54,
-                "score_ci_low": 0.34,
+                "score": 0.45,
+                "score_ci_high": 0.55,
+                "score_ci_low": 0.35,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.77,
-                "accuracy_ci_low": 0.68,
-                "accuracy_ci_high": 0.84,
+                "accuracy": 0.79,
+                "accuracy_ci_low": 0.7,
+                "accuracy_ci_high": 0.86,
                 "score_name": "accuracy",
-                "score": 0.77,
-                "score_ci_high": 0.84,
-                "score_ci_low": 0.68,
+                "score": 0.79,
+                "score_ci_high": 0.86,
+                "score_ci_low": 0.7,
                 "num_of_instances": 100
             },
-            "score": 0.605,
+            "score": 0.62,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -739,68 +739,68 @@
             "attaq_500": {
                 "severity_critical": 0.0,
                 "severity_high": 0.0,
-                "severity_medium": 0.0,
-                "severity_low": 100.0,
-                "category_harmful_info": 0.88618620190726,
-                "category_deception": 0.884011988311705,
-                "category_explicit_content": 0.875875836369967,
-                "category_violence": 0.8561706120281348,
-                "category_discrimination": 0.8707959112666902,
-                "category_substance_abuse": 0.9177011860741509,
-                "category_pii": 0.8376111410282275,
-                "safety": 0.8733486187820867,
-                "safety_ci_low": 0.8547795191133338,
-                "safety_ci_high": 0.8926551329161522,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8888607305651975,
+                "category_deception": 0.9223287611296683,
+                "category_explicit_content": 0.8829418493039681,
+                "category_violence": 0.8598237158244638,
+                "category_discrimination": 0.9429672291236264,
+                "category_substance_abuse": 0.9277940973639488,
+                "category_pii": 0.7872623268652844,
+                "safety": 0.8844143529573273,
+                "safety_ci_low": 0.8647379074366862,
+                "safety_ci_high": 0.9028506428126294,
                 "score_name": "safety",
-                "score": 0.8733486187820867,
-                "score_ci_high": 0.8926551329161522,
-                "score_ci_low": 0.8547795191133338,
+                "score": 0.8844143529573273,
+                "score_ci_high": 0.9028506428126294,
+                "score_ci_low": 0.8647379074366862,
                 "num_of_instances": 100
             },
-            "score": 0.8733486187820867,
+            "score": 0.8844143529573273,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.4108254867774814,
-                "rougeLsum": 0.34624370854410813,
-                "rougeL": 0.274953161069411,
-                "score": 0.274953161069411,
+                "rougeLsum": 0.3382265627183304,
+                "rouge1": 0.40036021824324175,
+                "rougeL": 0.26999463096681825,
+                "score": 0.26999463096681825,
                 "score_name": "rougeL",
-                "rouge2": 0.18851756344523637,
-                "rouge1_ci_low": 0.38844997336955545,
-                "rouge1_ci_high": 0.4278350029589772,
-                "rougeLsum_ci_low": 0.3268468521409277,
-                "rougeLsum_ci_high": 0.3627703780908734,
-                "rougeL_ci_low": 0.25981400465541243,
-                "rougeL_ci_high": 0.29002197762685467,
-                "score_ci_low": 0.25981400465541243,
-                "score_ci_high": 0.29002197762685467,
-                "rouge2_ci_low": 0.173970913148205,
-                "rouge2_ci_high": 0.20170296579389058
+                "rouge2": 0.1799170051796286,
+                "rougeLsum_ci_low": 0.3196770761195946,
+                "rougeLsum_ci_high": 0.35575095728361106,
+                "rouge1_ci_low": 0.37914911716209904,
+                "rouge1_ci_high": 0.4203788141262886,
+                "rougeL_ci_low": 0.2548331249292397,
+                "rougeL_ci_high": 0.2846240200157709,
+                "score_ci_low": 0.2548331249292397,
+                "score_ci_high": 0.2846240200157709,
+                "rouge2_ci_low": 0.16577019912913937,
+                "rouge2_ci_high": 0.19443334643705076
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.1024602229602419,
-                "rougeLsum": 0.08360059229921889,
-                "rougeL": 0.0729792585393188,
-                "score": 0.0729792585393188,
+                "rougeLsum": 0.08235468484076217,
+                "rouge1": 0.09928048902708886,
+                "rougeL": 0.07296386769372697,
+                "score": 0.07296386769372697,
                 "score_name": "rougeL",
-                "rouge2": 0.012617308836505319,
-                "rouge1_ci_low": 0.08836908504514274,
-                "rouge1_ci_high": 0.11693596916714544,
-                "rougeLsum_ci_low": 0.07226416836920484,
-                "rougeLsum_ci_high": 0.09465942240087559,
-                "rougeL_ci_low": 0.06348688700968229,
-                "rougeL_ci_high": 0.08263629933143575,
-                "score_ci_low": 0.06348688700968229,
-                "score_ci_high": 0.08263629933143575,
-                "rouge2_ci_low": 0.008790586379610133,
-                "rouge2_ci_high": 0.016979532193850944
+                "rouge2": 0.011236459579717977,
+                "rougeLsum_ci_low": 0.0722334775173192,
+                "rougeLsum_ci_high": 0.09370524228043663,
+                "rouge1_ci_low": 0.08657767471804466,
+                "rouge1_ci_high": 0.11349703017155294,
+                "rougeL_ci_low": 0.06434630685946219,
+                "rougeL_ci_high": 0.0827408872646653,
+                "score_ci_low": 0.06434630685946219,
+                "score_ci_high": 0.0827408872646653,
+                "rouge2_ci_low": 0.007906623743969794,
+                "rouge2_ci_high": 0.015085372921687724
             },
-            "score": 0.1739662098043649,
+            "score": 0.17147924933027262,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    120,
-                    72,
-                    51,
-                    37
+                    140,
+                    89,
+                    63,
+                    46
                 ],
                 "totals": [
-                    261,
-                    255,
-                    249,
-                    243
+                    253,
+                    247,
+                    241,
+                    235
                 ],
                 "precisions": [
-                    0.45977011494252873,
-                    0.2823529411764706,
-                    0.20481927710843373,
-                    0.1522633744855967
+                    0.5533596837944664,
+                    0.36032388663967607,
+                    0.26141078838174275,
+                    0.1957446808510638
                 ],
                 "bp": 1.0,
-                "sys_len": 261,
+                "sys_len": 253,
                 "ref_len": 208,
-                "sacrebleu": 0.25224631679056625,
-                "score": 0.25224631679056625,
+                "sacrebleu": 0.31781801514261865,
+                "score": 0.31781801514261865,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09290380581633799,
-                "score_ci_high": 0.44320740082704274,
-                "sacrebleu_ci_low": 0.09290380581633799,
-                "sacrebleu_ci_high": 0.44320740082704274
+                "score_ci_low": 0.22394855199440017,
+                "score_ci_high": 0.43009478960052605,
+                "sacrebleu_ci_low": 0.22394855199440017,
+                "sacrebleu_ci_high": 0.43009478960052605
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    134,
+                    137,
                     79,
-                    48,
-                    32
+                    46,
+                    27
                 ],
                 "totals": [
-                    372,
-                    366,
-                    360,
-                    354
+                    580,
+                    574,
+                    568,
+                    562
                 ],
                 "precisions": [
-                    0.3602150537634409,
-                    0.21584699453551914,
-                    0.13333333333333333,
-                    0.0903954802259887
+                    0.23620689655172414,
+                    0.1376306620209059,
+                    0.08098591549295774,
+                    0.048042704626334524
                 ],
                 "bp": 1.0,
-                "sys_len": 372,
+                "sys_len": 580,
                 "ref_len": 208,
-                "sacrebleu": 0.17496385110101034,
-                "score": 0.17496385110101034,
+                "sacrebleu": 0.10605012365966077,
+                "score": 0.10605012365966077,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10837277107576634,
-                "score_ci_high": 0.3346031415238713,
-                "sacrebleu_ci_low": 0.10837277107576634,
-                "sacrebleu_ci_high": 0.3346031415238713
+                "score_ci_low": 0.045773812142291564,
+                "score_ci_high": 0.2668828055491288,
+                "sacrebleu_ci_low": 0.045773812142291564,
+                "sacrebleu_ci_high": 0.2668828055491288
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    103,
-                    53,
-                    32,
-                    17
+                    95,
+                    42,
+                    25,
+                    15
                 ],
                 "totals": [
-                    210,
-                    204,
-                    198,
-                    192
+                    273,
+                    267,
+                    261,
+                    255
                 ],
                 "precisions": [
-                    0.4904761904761905,
-                    0.25980392156862747,
-                    0.16161616161616163,
-                    0.08854166666666666
+                    0.34798534798534797,
+                    0.15730337078651685,
+                    0.09578544061302682,
+                    0.05882352941176471
                 ],
                 "bp": 1.0,
-                "sys_len": 210,
+                "sys_len": 273,
                 "ref_len": 209,
-                "sacrebleu": 0.20664458446938855,
-                "score": 0.20664458446938855,
+                "sacrebleu": 0.13252182004183885,
+                "score": 0.13252182004183885,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.13004548257791534,
-                "score_ci_high": 0.2853017991472952,
-                "sacrebleu_ci_low": 0.13004548257791534,
-                "sacrebleu_ci_high": 0.2853017991472952
+                "score_ci_low": 0.07355875650706192,
+                "score_ci_high": 0.22875474913224045,
+                "sacrebleu_ci_low": 0.07355875650706192,
+                "sacrebleu_ci_high": 0.22875474913224045
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    127,
-                    74,
-                    43,
-                    25
+                    128,
+                    79,
+                    51,
+                    34
                 ],
                 "totals": [
-                    268,
-                    262,
-                    256,
-                    250
+                    303,
+                    297,
+                    291,
+                    285
                 ],
                 "precisions": [
-                    0.47388059701492535,
-                    0.2824427480916031,
-                    0.16796875,
-                    0.1
+                    0.42244224422442245,
+                    0.265993265993266,
+                    0.1752577319587629,
+                    0.11929824561403508
                 ],
                 "bp": 1.0,
-                "sys_len": 268,
+                "sys_len": 303,
                 "ref_len": 216,
-                "sacrebleu": 0.21774939719310438,
-                "score": 0.21774939719310438,
+                "sacrebleu": 0.22015943743267072,
+                "score": 0.22015943743267072,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10939756268894749,
-                "score_ci_high": 0.39371130045250813,
-                "sacrebleu_ci_low": 0.10939756268894749,
-                "sacrebleu_ci_high": 0.39371130045250813
+                "score_ci_low": 0.08834867956773865,
+                "score_ci_high": 0.39167266319022087,
+                "sacrebleu_ci_low": 0.08834867956773865,
+                "sacrebleu_ci_high": 0.39167266319022087
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    166,
-                    120,
-                    88,
-                    67
+                    171,
+                    116,
+                    83,
+                    61
                 ],
                 "totals": [
-                    251,
-                    245,
-                    239,
-                    233
+                    319,
+                    313,
+                    307,
+                    301
                 ],
                 "precisions": [
-                    0.6613545816733069,
-                    0.4897959183673469,
-                    0.36820083682008364,
-                    0.28755364806866957
+                    0.5360501567398119,
+                    0.3706070287539936,
+                    0.2703583061889251,
+                    0.2026578073089701
                 ],
                 "bp": 1.0,
-                "sys_len": 251,
+                "sys_len": 319,
                 "ref_len": 235,
-                "sacrebleu": 0.43034156537119933,
-                "score": 0.43034156537119933,
+                "sacrebleu": 0.3230022397669452,
+                "score": 0.3230022397669452,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.29544019591528686,
-                "score_ci_high": 0.5340905539945087,
-                "sacrebleu_ci_low": 0.29544019591528686,
-                "sacrebleu_ci_high": 0.5340905539945087
+                "score_ci_low": 0.2190080228024859,
+                "score_ci_high": 0.4140696646006612,
+                "sacrebleu_ci_low": 0.2190080228024859,
+                "sacrebleu_ci_high": 0.4140696646006612
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    115,
-                    53,
-                    31,
-                    18
+                    130,
+                    52,
+                    24,
+                    14
                 ],
                 "totals": [
-                    353,
-                    347,
-                    341,
-                    335
+                    433,
+                    427,
+                    421,
+                    415
                 ],
                 "precisions": [
-                    0.32577903682719545,
-                    0.1527377521613833,
-                    0.09090909090909091,
-                    0.053731343283582096
+                    0.3002309468822171,
+                    0.12177985948477751,
+                    0.057007125890736345,
+                    0.033734939759036145
                 ],
                 "bp": 1.0,
-                "sys_len": 353,
+                "sys_len": 433,
                 "ref_len": 249,
-                "sacrebleu": 0.12486080646404547,
-                "score": 0.12486080646404547,
+                "sacrebleu": 0.09157143128018339,
+                "score": 0.09157143128018339,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.058169488699475094,
-                "score_ci_high": 0.22645765592457381,
-                "sacrebleu_ci_low": 0.058169488699475094,
-                "sacrebleu_ci_high": 0.22645765592457381
+                "score_ci_low": 0.04232364990047081,
+                "score_ci_high": 0.12977264737145106,
+                "sacrebleu_ci_low": 0.04232364990047081,
+                "sacrebleu_ci_high": 0.12977264737145106
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    167,
-                    117,
-                    86,
-                    64
+                    161,
+                    115,
+                    88,
+                    67
                 ],
                 "totals": [
-                    947,
-                    941,
-                    935,
-                    929
+                    287,
+                    281,
+                    275,
+                    269
                 ],
                 "precisions": [
-                    0.17634635691657866,
-                    0.12433581296493092,
-                    0.09197860962566845,
-                    0.0688912809472551
+                    0.5609756097560975,
+                    0.4092526690391459,
+                    0.32,
+                    0.24907063197026022
                 ],
                 "bp": 1.0,
-                "sys_len": 947,
+                "sys_len": 287,
                 "ref_len": 222,
-                "sacrebleu": 0.1085684050344697,
-                "score": 0.1085684050344697,
+                "sacrebleu": 0.3677917643240569,
+                "score": 0.3677917643240569,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.05762493821668087,
-                "score_ci_high": 0.39256037604318816,
-                "sacrebleu_ci_low": 0.05762493821668087,
-                "sacrebleu_ci_high": 0.39256037604318816
+                "score_ci_low": 0.24880375866631974,
+                "score_ci_high": 0.44529960530247664,
+                "sacrebleu_ci_low": 0.24880375866631974,
+                "sacrebleu_ci_high": 0.44529960530247664
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    116,
-                    53,
-                    29,
+                    113,
+                    50,
+                    27,
                     17
                 ],
                 "totals": [
-                    234,
-                    228,
-                    222,
-                    216
+                    323,
+                    317,
+                    311,
+                    305
                 ],
                 "precisions": [
-                    0.49572649572649574,
-                    0.2324561403508772,
-                    0.13063063063063063,
-                    0.0787037037037037
+                    0.34984520123839014,
+                    0.15772870662460567,
+                    0.08681672025723472,
+                    0.05573770491803279
                 ],
                 "bp": 1.0,
-                "sys_len": 234,
+                "sys_len": 323,
                 "ref_len": 230,
-                "sacrebleu": 0.1855264510586811,
-                "score": 0.1855264510586811,
+                "sacrebleu": 0.1278305860866586,
+                "score": 0.1278305860866586,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10495507855769916,
-                "score_ci_high": 0.3369225105649729,
-                "sacrebleu_ci_low": 0.10495507855769916,
-                "sacrebleu_ci_high": 0.3369225105649729
+                "score_ci_low": 0.043467722634347436,
+                "score_ci_high": 0.28760202689857367,
+                "sacrebleu_ci_low": 0.043467722634347436,
+                "sacrebleu_ci_high": 0.28760202689857367
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    149,
-                    79,
-                    46,
-                    27
+                    150,
+                    75,
+                    39,
+                    23
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    275,
+                    269,
+                    263,
+                    257
                 ],
                 "precisions": [
-                    0.6622222222222223,
-                    0.36073059360730597,
-                    0.215962441314554,
-                    0.13043478260869565
+                    0.5454545454545454,
+                    0.2788104089219331,
+                    0.1482889733840304,
+                    0.08949416342412451
                 ],
-                "bp": 0.9231163463866358,
-                "sys_len": 225,
+                "bp": 1.0,
+                "sys_len": 275,
                 "ref_len": 243,
-                "sacrebleu": 0.26439061488725174,
-                "score": 0.26439061488725174,
+                "sacrebleu": 0.21195456757616282,
+                "score": 0.21195456757616282,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23353321668371868,
-                "score_ci_high": 0.2896591819189158,
-                "sacrebleu_ci_low": 0.23353321668371868,
-                "sacrebleu_ci_high": 0.2896591819189158
+                "score_ci_low": 0.1231274458577025,
+                "score_ci_high": 0.2592759014746836,
+                "sacrebleu_ci_low": 0.1231274458577025,
+                "sacrebleu_ci_high": 0.2592759014746836
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    156,
-                    108,
-                    76,
-                    55
+                    148,
+                    97,
+                    65,
+                    43
                 ],
                 "totals": [
-                    449,
-                    443,
-                    437,
-                    431
+                    294,
+                    288,
+                    282,
+                    276
                 ],
                 "precisions": [
-                    0.34743875278396436,
-                    0.24379232505643342,
-                    0.17391304347826086,
-                    0.12761020881670534
+                    0.5034013605442177,
+                    0.3368055555555556,
+                    0.23049645390070922,
+                    0.15579710144927536
                 ],
                 "bp": 1.0,
-                "sys_len": 449,
+                "sys_len": 294,
                 "ref_len": 208,
-                "sacrebleu": 0.20822312752161715,
-                "score": 0.20822312752161715,
+                "sacrebleu": 0.2793375458793261,
+                "score": 0.2793375458793261,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08646910197241711,
-                "score_ci_high": 0.4918866743674902,
-                "sacrebleu_ci_low": 0.08646910197241711,
-                "sacrebleu_ci_high": 0.4918866743674902
+                "score_ci_low": 0.14985482540315964,
+                "score_ci_high": 0.4394380580658596,
+                "sacrebleu_ci_low": 0.14985482540315964,
+                "sacrebleu_ci_high": 0.4394380580658596
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    125,
-                    57,
-                    33,
-                    20
+                    131,
+                    71,
+                    42,
+                    25
                 ],
                 "totals": [
-                    386,
-                    380,
-                    374,
-                    368
+                    308,
+                    302,
+                    296,
+                    290
                 ],
                 "precisions": [
-                    0.32383419689119175,
-                    0.15,
-                    0.08823529411764706,
-                    0.05434782608695652
+                    0.4253246753246753,
+                    0.23509933774834438,
+                    0.14189189189189189,
+                    0.08620689655172414
                 ],
                 "bp": 1.0,
-                "sys_len": 386,
+                "sys_len": 308,
                 "ref_len": 208,
-                "sacrebleu": 0.12354057561268782,
-                "score": 0.12354057561268782,
+                "sacrebleu": 0.1870113191838078,
+                "score": 0.1870113191838078,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08093532121195993,
-                "score_ci_high": 0.1474721378465118,
-                "sacrebleu_ci_low": 0.08093532121195993,
-                "sacrebleu_ci_high": 0.1474721378465118
+                "score_ci_low": 0.09322821548809718,
+                "score_ci_high": 0.24013861737027561,
+                "sacrebleu_ci_low": 0.09322821548809718,
+                "sacrebleu_ci_high": 0.24013861737027561
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    124,
-                    69,
-                    40,
-                    26
+                    115,
+                    53,
+                    24,
+                    12
                 ],
                 "totals": [
-                    261,
-                    255,
-                    249,
-                    243
+                    335,
+                    329,
+                    323,
+                    317
                 ],
                 "precisions": [
-                    0.47509578544061304,
-                    0.27058823529411763,
-                    0.1606425702811245,
-                    0.10699588477366255
+                    0.34328358208955223,
+                    0.16109422492401215,
+                    0.07430340557275542,
+                    0.03785488958990536
                 ],
                 "bp": 1.0,
-                "sys_len": 261,
+                "sys_len": 335,
                 "ref_len": 208,
-                "sacrebleu": 0.21681007101221858,
-                "score": 0.21681007101221858,
+                "sacrebleu": 0.11167756267294884,
+                "score": 0.11167756267294884,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1453167229499001,
-                "score_ci_high": 0.2575611599258459,
-                "sacrebleu_ci_low": 0.1453167229499001,
-                "sacrebleu_ci_high": 0.2575611599258459
+                "score_ci_low": 0.0750863466115135,
+                "score_ci_high": 0.1511603264034476,
+                "sacrebleu_ci_low": 0.0750863466115135,
+                "sacrebleu_ci_high": 0.1511603264034476
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    101,
-                    68,
-                    47
+                    156,
+                    113,
+                    85,
+                    66
                 ],
                 "totals": [
-                    298,
-                    292,
-                    286,
-                    280
+                    300,
+                    294,
+                    288,
+                    282
                 ],
                 "precisions": [
-                    0.5067114093959731,
-                    0.3458904109589041,
-                    0.23776223776223776,
-                    0.16785714285714284
+                    0.52,
+                    0.3843537414965986,
+                    0.2951388888888889,
+                    0.23404255319148937
                 ],
                 "bp": 1.0,
-                "sys_len": 298,
+                "sys_len": 300,
                 "ref_len": 208,
-                "sacrebleu": 0.2891981283448437,
-                "score": 0.2891981283448437,
+                "sacrebleu": 0.34277878137474943,
+                "score": 0.34277878137474943,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.22977097370221483,
-                "score_ci_high": 0.34262336728855886,
-                "sacrebleu_ci_low": 0.22977097370221483,
-                "sacrebleu_ci_high": 0.34262336728855886
+                "score_ci_low": 0.19792804696545602,
+                "score_ci_high": 0.4586552468500489,
+                "sacrebleu_ci_low": 0.19792804696545602,
+                "sacrebleu_ci_high": 0.4586552468500489
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    94,
-                    60,
-                    38
+                    158,
+                    95,
+                    66,
+                    50
                 ],
                 "totals": [
-                    344,
-                    338,
-                    332,
-                    326
+                    582,
+                    576,
+                    570,
+                    564
                 ],
                 "precisions": [
-                    0.438953488372093,
-                    0.2781065088757396,
-                    0.18072289156626506,
-                    0.11656441717791412
+                    0.27147766323024053,
+                    0.16493055555555558,
+                    0.11578947368421053,
+                    0.08865248226950355
                 ],
                 "bp": 1.0,
-                "sys_len": 344,
+                "sys_len": 582,
                 "ref_len": 208,
-                "sacrebleu": 0.22519160969750285,
-                "score": 0.22519160969750285,
+                "sacrebleu": 0.14641946009125262,
+                "score": 0.14641946009125262,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09845675192327814,
-                "score_ci_high": 0.35805619211106726,
-                "sacrebleu_ci_low": 0.09845675192327814,
-                "sacrebleu_ci_high": 0.35805619211106726
+                "score_ci_low": 0.047945874214537776,
+                "score_ci_high": 0.325694681550346,
+                "sacrebleu_ci_low": 0.047945874214537776,
+                "sacrebleu_ci_high": 0.325694681550346
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    98,
-                    68,
-                    46
+                    142,
+                    87,
+                    58,
+                    39
                 ],
                 "totals": [
-                    360,
-                    354,
-                    348,
-                    342
+                    241,
+                    235,
+                    229,
+                    223
                 ],
                 "precisions": [
-                    0.41944444444444445,
-                    0.2768361581920904,
-                    0.1954022988505747,
-                    0.13450292397660818
+                    0.5892116182572614,
+                    0.3702127659574468,
+                    0.2532751091703057,
+                    0.17488789237668162
                 ],
                 "bp": 1.0,
-                "sys_len": 360,
+                "sys_len": 241,
                 "ref_len": 208,
-                "sacrebleu": 0.23503885547291925,
-                "score": 0.23503885547291925,
+                "sacrebleu": 0.31352251684251853,
+                "score": 0.31352251684251853,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10883001842131954,
-                "score_ci_high": 0.39236450813500995,
-                "sacrebleu_ci_low": 0.10883001842131954,
-                "sacrebleu_ci_high": 0.39236450813500995
+                "score_ci_low": 0.16226292142208393,
+                "score_ci_high": 0.3894343521622631,
+                "sacrebleu_ci_low": 0.16226292142208393,
+                "sacrebleu_ci_high": 0.3894343521622631
             },
-            "score": 0.21755295733543376,
+            "score": 0.21862981142369328,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.47041353678026604,
+        "score": 0.4928793375118048,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/2025-07-03T08-05-54_evaluation_results.json

@@ -0,0 +1,1281 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-07-03T12:05:51.695495Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/ibm/granite-3-8b-instruct,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/ibm/granite-3-8b-instruct",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.25.0",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "unitxt": "1.25.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "litellm": "1.73.6",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "tokenizers": "0.21.2",
+            "yarl": "1.20.1",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "openai": "1.93.0",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "pillow": "11.3.0",
+            "fonttools": "4.58.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "huggingface-hub": "0.33.2",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "accelerate": "1.8.1",
+            "certifi": "2025.6.15",
+            "evaluate": "0.4.4",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "transformers": "4.53.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.47716657027690984,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.47716657027690984,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5310928992288233,
+                "num_of_instances": 9
+            },
+            "score": 0.8383838383838383,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.5131578947368421,
+                "score": 0.5131578947368421,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.5131578947368421,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Location": 0.22727272727272727,
+                "f1_Person": 0.5217391304347826,
+                "f1_Organization": 0.4262295081967213,
+                "f1_macro": 0.39174712196807704,
+                "recall_macro": 0.39811939268461005,
+                "precision_macro": 0.3885595081247255,
+                "in_classes_support": 0.6178861788617886,
+                "f1_micro": 0.30303030303030304,
+                "recall_micro": 0.4,
+                "precision_micro": 0.24390243902439024,
+                "score": 0.30303030303030304,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.07496516805145101,
+                "score_ci_high": 0.42390724339941516,
+                "f1_micro_ci_low": 0.07496516805145101,
+                "f1_micro_ci_high": 0.42390724339941516
+            },
+            "score": 0.30303030303030304,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2530277506117974,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8571428571428571,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.42857142857142855,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7469722493882013,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.7469722493882013,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "score": 0.4387755102040816,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.43666666666666665,
+                "f1_suggestive": 0.5,
+                "f1_descriptive": 0.5333333333333333,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.4,
+                "f1_arbitrary": 0.75,
+                "f1_macro_ci_low": 0.2596895818708838,
+                "f1_macro_ci_high": 0.7109644675737796,
+                "score_name": "f1_micro",
+                "score": 0.5128205128205128,
+                "score_ci_high": 0.7295250374354838,
+                "score_ci_low": 0.2683716341971827,
+                "num_of_instances": 20,
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5128205128205128,
+                "f1_micro_ci_low": 0.2683716341971827,
+                "f1_micro_ci_high": 0.7295250374354838
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.4982078853046595,
+                "f1_no": 0.7741935483870968,
+                "f1_yes": 0.2222222222222222,
+                "f1_macro_ci_low": 0.3548387096774194,
+                "f1_macro_ci_high": 0.918918918918919,
+                "score_name": "f1_micro",
+                "score": 0.65,
+                "score_ci_high": 0.85,
+                "score_ci_low": 0.4,
+                "num_of_instances": 20,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.65,
+                "f1_micro_ci_low": 0.4,
+                "f1_micro_ci_high": 0.85
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.06746031746031746,
+                "f1_conclusion": 0.25,
+                "f1_issue": 0.2222222222222222,
+                "f1_decree": 0.0,
+                "f1_rule": 0.0,
+                "f1_analysis": 0.0,
+                "f1_facts": 0.0,
+                "f1_procedural history": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.19797979797979798,
+                "score_name": "f1_micro",
+                "score": 0.1,
+                "score_ci_high": 0.2777777777777778,
+                "score_ci_low": 0.0,
+                "num_of_instances": 20,
+                "accuracy": 0.1,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.3,
+                "f1_micro": 0.1,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.2777777777777778
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.31666666666666665,
+                "f1_yes": 0.5,
+                "f1_no": 0.13333333333333333,
+                "f1_macro_ci_low": 0.19184876549102856,
+                "f1_macro_ci_high": 0.5562869410797749,
+                "score_name": "f1_micro",
+                "score": 0.358974358974359,
+                "score_ci_high": 0.5984946237513465,
+                "score_ci_low": 0.15789473684210525,
+                "num_of_instances": 20,
+                "accuracy": 0.35,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.358974358974359,
+                "f1_micro_ci_low": 0.15789473684210525,
+                "f1_micro_ci_high": 0.5984946237513465
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.8705882352941177,
+                "f1_yes": 0.9411764705882353,
+                "f1_no": 0.8,
+                "f1_macro_ci_low": 0.6705536779335216,
+                "f1_macro_ci_high": 0.9615384615384616,
+                "score_name": "f1_micro",
+                "score": 0.8648648648648649,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.6502941649122055,
+                "num_of_instances": 20,
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.5720066230431405,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.8648648648648649,
+                "f1_micro_ci_low": 0.6502941649122055,
+                "f1_micro_ci_high": 0.95
+            },
+            "score": 0.49733194733194735,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.4239959524479648,
+                "f1_cars": 0.8,
+                "f1_pc hardware": 0.5454545454545454,
+                "f1_windows x": 0.0,
+                "f1_computer graphics": 0.5882352941176471,
+                "f1_atheism": 0.0,
+                "f1_religion": 0.18181818181818182,
+                "f1_medicine": 0.6666666666666666,
+                "f1_christianity": 0.0,
+                "f1_for sale": 0.6,
+                "f1_microsoft windows": 0.5,
+                "f1_middle east": 0.0,
+                "f1_motorcycles": 0.6,
+                "f1_mac hardware": 0.3333333333333333,
+                "f1_guns": 0.5,
+                "f1_politics": 0.5263157894736842,
+                "f1_space": 0.5714285714285714,
+                "f1_cryptography": 0.4,
+                "f1_hockey": 0.6666666666666666,
+                "f1_baseball": 1.0,
+                "f1_electronics": 0.0,
+                "f1_macro_ci_low": 0.34375053675075556,
+                "f1_macro_ci_high": 0.5229994216966102,
+                "score_name": "f1_micro",
+                "score": 0.5,
+                "score_ci_high": 0.6010991243020599,
+                "score_ci_low": 0.3954802259887006,
+                "num_of_instances": 100,
+                "accuracy": 0.46,
+                "accuracy_ci_low": 0.36,
+                "accuracy_ci_high": 0.56,
+                "f1_micro": 0.5,
+                "f1_micro_ci_low": 0.3954802259887006,
+                "f1_micro_ci_high": 0.6010991243020599
+            },
+            "score": 0.5,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7207043858492499,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9253731343283582,
+                "f1_money transfer or virtual currency or money service": 0.8,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_credit card or prepaid card": 0.7058823529411765,
+                "f1_debt collection": 0.7777777777777778,
+                "f1_checking or savings account": 0.7692307692307693,
+                "f1_payday loan or title loan or personal loan": 0.4,
+                "f1_macro_ci_low": 0.4815679833092744,
+                "f1_macro_ci_high": 0.8364352565324512,
+                "score_name": "f1_micro",
+                "score": 0.8615384615384616,
+                "score_ci_high": 0.9137055837563451,
+                "score_ci_low": 0.7635197536237738,
+                "num_of_instances": 100,
+                "accuracy": 0.84,
+                "accuracy_ci_low": 0.75,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.8615384615384616,
+                "f1_micro_ci_low": 0.7635197536237738,
+                "f1_micro_ci_high": 0.9137055837563451
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.6270669427191166,
+                "f1_mortgages and loans": 0.7619047619047619,
+                "f1_credit card": 0.5,
+                "f1_credit reporting": 0.6956521739130435,
+                "f1_debt collection": 0.7777777777777778,
+                "f1_retail banking": 0.4,
+                "f1_macro_ci_low": 0.4966140372134347,
+                "f1_macro_ci_high": 0.7555085024990378,
+                "score_name": "f1_micro",
+                "score": 0.6391752577319587,
+                "score_ci_high": 0.7609225921717959,
+                "score_ci_low": 0.5,
+                "num_of_instances": 50,
+                "accuracy": 0.62,
+                "accuracy_ci_low": 0.48,
+                "accuracy_ci_high": 0.74,
+                "f1_micro": 0.6391752577319587,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.7609225921717959
+            },
+            "score": 0.7503568596352101,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "execution_accuracy": 0.15,
+                "program_accuracy": 0.16,
+                "score": 0.16,
+                "score_name": "program_accuracy",
+                "execution_accuracy_ci_low": 0.09,
+                "execution_accuracy_ci_high": 0.23571967026025617,
+                "program_accuracy_ci_low": 0.1,
+                "program_accuracy_ci_high": 0.24,
+                "score_ci_low": 0.1,
+                "score_ci_high": 0.24
+            },
+            "score": 0.16,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.5212876936378656,
+                "recall": 0.6578052424371036,
+                "f1": 0.5396867565070588,
+                "precision_ci_low": 0.4809046415775284,
+                "precision_ci_high": 0.5620677262313715,
+                "recall_ci_low": 0.6172772682646432,
+                "recall_ci_high": 0.6934399487258313,
+                "f1_ci_low": 0.5109945288978699,
+                "f1_ci_high": 0.5735415710358195,
+                "score_name": "f1",
+                "score": 0.5396867565070588,
+                "score_ci_high": 0.5735415710358195,
+                "score_ci_low": 0.5109945288978699,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.7192348712682723,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7497482949495315,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.7014025217294693,
+                "faithfullness_f1_token_overlap": 0.47668538259414794,
+                "faithfullness_recall_token_overlap": 0.37725895304401214,
+                "faithfullness_precision_token_overlap": 0.792319814348149,
+                "correctness_f1_token_overlap": 0.5396867565070588,
+                "correctness_recall_token_overlap": 0.6578052424371036,
+                "correctness_precision_token_overlap": 0.5212876936378656
+            },
+            "score": 0.5396867565070588,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.46,
+                "accuracy_ci_low": 0.3643470807957145,
+                "accuracy_ci_high": 0.5596197667548154,
+                "score_name": "accuracy",
+                "score": 0.46,
+                "score_ci_high": 0.5596197667548154,
+                "score_ci_low": 0.3643470807957145,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.85,
+                "accuracy_ci_low": 0.77,
+                "accuracy_ci_high": 0.91,
+                "score_name": "accuracy",
+                "score": 0.85,
+                "score_ci_high": 0.91,
+                "score_ci_low": 0.77,
+                "num_of_instances": 100
+            },
+            "score": 0.655,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8925065078314335,
+                "category_deception": 0.9381265209028213,
+                "category_explicit_content": 0.8503636385154242,
+                "category_violence": 0.8601648936432505,
+                "category_discrimination": 0.8948557959307754,
+                "category_substance_abuse": 0.9021706441210376,
+                "category_pii": 0.8278552542130152,
+                "safety": 0.8802775480515426,
+                "safety_ci_low": 0.8583081755069534,
+                "safety_ci_high": 0.9001231887506078,
+                "score_name": "safety",
+                "score": 0.8802775480515426,
+                "score_ci_high": 0.9001231887506078,
+                "score_ci_low": 0.8583081755069534,
+                "num_of_instances": 100
+            },
+            "score": 0.8802775480515426,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeL": 0.283793962737229,
+                "score": 0.283793962737229,
+                "score_name": "rougeL",
+                "rougeLsum": 0.35399993809096103,
+                "rouge1": 0.41485236441679585,
+                "rouge2": 0.19594736920885997,
+                "rougeL_ci_low": 0.26505929835848524,
+                "rougeL_ci_high": 0.30137043303436234,
+                "score_ci_low": 0.26505929835848524,
+                "score_ci_high": 0.30137043303436234,
+                "rougeLsum_ci_low": 0.3318357775824492,
+                "rougeLsum_ci_high": 0.37317376467108154,
+                "rouge1_ci_low": 0.38944020954394787,
+                "rouge1_ci_high": 0.4348179076446575,
+                "rouge2_ci_low": 0.17823964075516807,
+                "rouge2_ci_high": 0.2137693144513423
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rougeL": 0.07662085019609605,
+                "score": 0.07662085019609605,
+                "score_name": "rougeL",
+                "rougeLsum": 0.0861008326275811,
+                "rouge1": 0.10327424535868773,
+                "rouge2": 0.013891283076053876,
+                "rougeL_ci_low": 0.06625536309669923,
+                "rougeL_ci_high": 0.08742816549707229,
+                "score_ci_low": 0.06625536309669923,
+                "score_ci_high": 0.08742816549707229,
+                "rougeLsum_ci_low": 0.0747762244390909,
+                "rougeLsum_ci_high": 0.09831815623274179,
+                "rouge1_ci_low": 0.08913123349846178,
+                "rouge1_ci_high": 0.1189079844960333,
+                "rouge2_ci_low": 0.009795628030972649,
+                "rouge2_ci_high": 0.019088127419088847
+            },
+            "score": 0.18020740646666253,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    144,
+                    91,
+                    62,
+                    43
+                ],
+                "totals": [
+                    702,
+                    696,
+                    690,
+                    684
+                ],
+                "precisions": [
+                    0.20512820512820515,
+                    0.1307471264367816,
+                    0.08985507246376813,
+                    0.06286549707602339
+                ],
+                "bp": 1.0,
+                "sys_len": 702,
+                "ref_len": 208,
+                "sacrebleu": 0.11094382152385164,
+                "score": 0.11094382152385164,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.0430216194039613,
+                "score_ci_high": 0.4258937195147393,
+                "sacrebleu_ci_low": 0.0430216194039613,
+                "sacrebleu_ci_high": 0.4258937195147393
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    135,
+                    83,
+                    52,
+                    32
+                ],
+                "totals": [
+                    303,
+                    297,
+                    291,
+                    285
+                ],
+                "precisions": [
+                    0.44554455445544555,
+                    0.27946127946127947,
+                    0.17869415807560138,
+                    0.11228070175438595
+                ],
+                "bp": 1.0,
+                "sys_len": 303,
+                "ref_len": 208,
+                "sacrebleu": 0.22356667304067482,
+                "score": 0.22356667304067482,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.08578859861530164,
+                "score_ci_high": 0.4405729805595396,
+                "sacrebleu_ci_low": 0.08578859861530164,
+                "sacrebleu_ci_high": 0.4405729805595396
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    99,
+                    40,
+                    18,
+                    9
+                ],
+                "totals": [
+                    246,
+                    240,
+                    234,
+                    228
+                ],
+                "precisions": [
+                    0.40243902439024387,
+                    0.16666666666666669,
+                    0.07692307692307693,
+                    0.039473684210526314
+                ],
+                "bp": 1.0,
+                "sys_len": 246,
+                "ref_len": 209,
+                "sacrebleu": 0.11946158890640456,
+                "score": 0.11946158890640456,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.06603109486053045,
+                "score_ci_high": 0.16414996430005616,
+                "sacrebleu_ci_low": 0.06603109486053045,
+                "sacrebleu_ci_high": 0.16414996430005616
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    129,
+                    68,
+                    40,
+                    23
+                ],
+                "totals": [
+                    371,
+                    365,
+                    359,
+                    353
+                ],
+                "precisions": [
+                    0.3477088948787062,
+                    0.1863013698630137,
+                    0.11142061281337048,
+                    0.06515580736543909
+                ],
+                "bp": 1.0,
+                "sys_len": 371,
+                "ref_len": 216,
+                "sacrebleu": 0.1472609611005195,
+                "score": 0.1472609611005195,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.10737963346465397,
+                "score_ci_high": 0.22901389704600852,
+                "sacrebleu_ci_low": 0.10737963346465397,
+                "sacrebleu_ci_high": 0.22901389704600852
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    108,
+                    78,
+                    58
+                ],
+                "totals": [
+                    315,
+                    309,
+                    303,
+                    297
+                ],
+                "precisions": [
+                    0.5238095238095238,
+                    0.34951456310679613,
+                    0.25742574257425743,
+                    0.19528619528619529
+                ],
+                "bp": 1.0,
+                "sys_len": 315,
+                "ref_len": 235,
+                "sacrebleu": 0.3097351874987371,
+                "score": 0.3097351874987371,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19047973841407545,
+                "score_ci_high": 0.42375919110879456,
+                "sacrebleu_ci_low": 0.19047973841407545,
+                "sacrebleu_ci_high": 0.42375919110879456
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    139,
+                    68,
+                    41,
+                    25
+                ],
+                "totals": [
+                    374,
+                    368,
+                    362,
+                    356
+                ],
+                "precisions": [
+                    0.3716577540106952,
+                    0.1847826086956522,
+                    0.1132596685082873,
+                    0.07022471910112359
+                ],
+                "bp": 1.0,
+                "sys_len": 374,
+                "ref_len": 249,
+                "sacrebleu": 0.15287708643378742,
+                "score": 0.15287708643378742,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.09848546235015815,
+                "score_ci_high": 0.20815357764237413,
+                "sacrebleu_ci_low": 0.09848546235015815,
+                "sacrebleu_ci_high": 0.20815357764237413
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    170,
+                    124,
+                    102,
+                    85
+                ],
+                "totals": [
+                    601,
+                    595,
+                    589,
+                    583
+                ],
+                "precisions": [
+                    0.2828618968386023,
+                    0.20840336134453782,
+                    0.1731748726655348,
+                    0.14579759862778732
+                ],
+                "bp": 1.0,
+                "sys_len": 601,
+                "ref_len": 222,
+                "sacrebleu": 0.1964167877420127,
+                "score": 0.1964167877420127,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.07270540057393714,
+                "score_ci_high": 0.518665491959237,
+                "sacrebleu_ci_low": 0.07270540057393714,
+                "sacrebleu_ci_high": 0.518665491959237
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    117,
+                    56,
+                    35,
+                    24
+                ],
+                "totals": [
+                    377,
+                    371,
+                    365,
+                    359
+                ],
+                "precisions": [
+                    0.3103448275862069,
+                    0.1509433962264151,
+                    0.0958904109589041,
+                    0.06685236768802229
+                ],
+                "bp": 1.0,
+                "sys_len": 377,
+                "ref_len": 230,
+                "sacrebleu": 0.13163993236575622,
+                "score": 0.13163993236575622,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.046393246503418153,
+                "score_ci_high": 0.2006802995057917,
+                "sacrebleu_ci_low": 0.046393246503418153,
+                "sacrebleu_ci_high": 0.2006802995057917
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    159,
+                    89,
+                    56,
+                    35
+                ],
+                "totals": [
+                    353,
+                    347,
+                    341,
+                    335
+                ],
+                "precisions": [
+                    0.45042492917847027,
+                    0.2564841498559078,
+                    0.16422287390029325,
+                    0.10447761194029852
+                ],
+                "bp": 1.0,
+                "sys_len": 353,
+                "ref_len": 243,
+                "sacrebleu": 0.21100121642971456,
+                "score": 0.21100121642971456,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.15841899383396243,
+                "score_ci_high": 0.28733352469932727,
+                "sacrebleu_ci_low": 0.15841899383396243,
+                "sacrebleu_ci_high": 0.28733352469932727
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    161,
+                    110,
+                    82,
+                    61
+                ],
+                "totals": [
+                    403,
+                    397,
+                    391,
+                    385
+                ],
+                "precisions": [
+                    0.39950372208436724,
+                    0.2770780856423174,
+                    0.20971867007672634,
+                    0.15844155844155844
+                ],
+                "bp": 1.0,
+                "sys_len": 403,
+                "ref_len": 208,
+                "sacrebleu": 0.2462676137243911,
+                "score": 0.2462676137243911,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.11679608311837608,
+                "score_ci_high": 0.434165455858711,
+                "sacrebleu_ci_low": 0.11679608311837608,
+                "sacrebleu_ci_high": 0.434165455858711
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    121,
+                    64,
+                    38,
+                    21
+                ],
+                "totals": [
+                    270,
+                    264,
+                    258,
+                    252
+                ],
+                "precisions": [
+                    0.4481481481481482,
+                    0.24242424242424243,
+                    0.14728682170542634,
+                    0.08333333333333334
+                ],
+                "bp": 1.0,
+                "sys_len": 270,
+                "ref_len": 208,
+                "sacrebleu": 0.19109313021092122,
+                "score": 0.19109313021092122,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.0823173685610723,
+                "score_ci_high": 0.26847845369224815,
+                "sacrebleu_ci_low": 0.0823173685610723,
+                "sacrebleu_ci_high": 0.26847845369224815
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    118,
+                    51,
+                    24,
+                    10
+                ],
+                "totals": [
+                    295,
+                    289,
+                    283,
+                    277
+                ],
+                "precisions": [
+                    0.4,
+                    0.17647058823529413,
+                    0.08480565371024734,
+                    0.036101083032490974
+                ],
+                "bp": 1.0,
+                "sys_len": 295,
+                "ref_len": 208,
+                "sacrebleu": 0.12124653620698017,
+                "score": 0.12124653620698017,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.028053817072628193,
+                "score_ci_high": 0.22022584763455738,
+                "sacrebleu_ci_low": 0.028053817072628193,
+                "sacrebleu_ci_high": 0.22022584763455738
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    154,
+                    101,
+                    68,
+                    46
+                ],
+                "totals": [
+                    213,
+                    207,
+                    201,
+                    195
+                ],
+                "precisions": [
+                    0.7230046948356808,
+                    0.48792270531400966,
+                    0.33830845771144274,
+                    0.23589743589743592
+                ],
+                "bp": 1.0,
+                "sys_len": 213,
+                "ref_len": 208,
+                "sacrebleu": 0.4096208508449147,
+                "score": 0.4096208508449147,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.32602690169539184,
+                "score_ci_high": 0.5203750517974051,
+                "sacrebleu_ci_low": 0.32602690169539184,
+                "sacrebleu_ci_high": 0.5203750517974051
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    158,
+                    99,
+                    67,
+                    47
+                ],
+                "totals": [
+                    1096,
+                    1090,
+                    1084,
+                    1078
+                ],
+                "precisions": [
+                    0.14416058394160586,
+                    0.09082568807339449,
+                    0.061808118081180814,
+                    0.043599257884972174
+                ],
+                "bp": 1.0,
+                "sys_len": 1096,
+                "ref_len": 208,
+                "sacrebleu": 0.07707170412963754,
+                "score": 0.07707170412963754,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.032135248195696395,
+                "score_ci_high": 0.22292436255792658,
+                "sacrebleu_ci_low": 0.032135248195696395,
+                "sacrebleu_ci_high": 0.22292436255792658
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    98,
+                    68,
+                    46
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.7203791469194313,
+                    0.47804878048780486,
+                    0.3417085427135678,
+                    0.23834196891191708
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 208,
+                "sacrebleu": 0.40923467652036666,
+                "score": 0.40923467652036666,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2684007842080438,
+                "score_ci_high": 0.47744647601879564,
+                "sacrebleu_ci_low": 0.2684007842080438,
+                "sacrebleu_ci_high": 0.47744647601879564
+            },
+            "score": 0.20382918444524467,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.4969259422148255,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/{2025-07-02T16-08-27_evaluation_results.json → 2025-07-03T08-48-01_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T20:08:24.273547Z",
+        "timestamp_utc": "2025-07-03T12:47:54.386872Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,12 +176,12 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.4444444444444444,
+                "accuracy": 0.3333333333333333,
                 "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy_ci_high": 0.6666666666666666,
                 "score_name": "accuracy",
-                "score": 0.4444444444444444,
-                "score_ci_high": 0.7777777777777778,
+                "score": 0.3333333333333333,
+                "score_ci_high": 0.6666666666666666,
                 "score_ci_low": 0.1111111111111111,
                 "num_of_instances": 9
             },
@@ -196,23 +196,23 @@
                 "num_of_instances": 9
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.2222222222222222,
-                "accuracy_ci_high": 0.8888888888888888,
+                "accuracy": 0.3333333333333333,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.6666666666666666,
                 "score_name": "accuracy",
-                "score": 0.5555555555555556,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.2222222222222222,
+                "score": 0.3333333333333333,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.1111111111111111,
                 "num_of_instances": 9
             },
             "safety_bbq_nationality": {
-                "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.2222222222222222,
-                "accuracy_ci_high": 0.8888888888888888,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.5555555555555556,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.2222222222222222,
+                "score": 0.6666666666666666,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
             "safety_bbq_physical_appearance": {
@@ -226,13 +226,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.4444444444444444,
-                "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.4444444444444444,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.1111111111111111,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_gender": {
@@ -246,13 +246,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.4444444444444444,
-                "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.4444444444444444,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.1111111111111111,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_religion": {
@@ -292,35 +292,35 @@
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.07109004739336493,
-                "score": 0.07109004739336493,
+                "llama_3_70b_instruct_template_arena_hard": 0.05701754385964912,
+                "score": 0.05701754385964912,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.07109004739336493,
+            "score": 0.05701754385964912,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.3333333333333333,
-                "f1_Organization": 0.14285714285714285,
-                "f1_Location": 0.26666666666666666,
-                "f1_macro": 0.24761904761904763,
-                "recall_macro": 0.1637336093857833,
-                "precision_macro": 0.5317460317460317,
-                "in_classes_support": 0.7714285714285715,
-                "f1_micro": 0.21818181818181817,
-                "recall_micro": 0.16,
-                "precision_micro": 0.34285714285714286,
-                "score": 0.21818181818181817,
+                "f1_Person": 0.31999999999999995,
+                "f1_Organization": 0.21052631578947367,
+                "f1_Location": 0.2,
+                "f1_macro": 0.24350877192982456,
+                "recall_macro": 0.2429261559696342,
+                "precision_macro": 0.2510642826734781,
+                "in_classes_support": 0.7272727272727273,
+                "f1_micro": 0.20689655172413793,
+                "recall_micro": 0.24,
+                "precision_micro": 0.18181818181818182,
+                "score": 0.20689655172413793,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.10426925532422195,
-                "score_ci_high": 0.3444160571809249,
-                "f1_micro_ci_low": 0.10426925532422195,
-                "f1_micro_ci_high": 0.3444160571809249
+                "score_ci_low": 0.15899844152563253,
+                "score_ci_high": 0.3195980554375623,
+                "f1_micro_ci_low": 0.15899844152563253,
+                "f1_micro_ci_high": 0.3195980554375623
             },
-            "score": 0.21818181818181817,
+            "score": 0.20689655172413793,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -338,31 +338,31 @@
             "mmlu_pro_business": {
                 "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.6807203593841678,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
                 "score": 0.14285714285714285,
-                "score_ci_high": 0.6807203593841678,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.2857142857142857,
+                "accuracy": 0.0,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy_ci_high": 0.0,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
+                "score": 0.0,
+                "score_ci_high": 0.0,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_economics": {
@@ -386,12 +386,12 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.0,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.0,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.0,
+                "score_ci_high": 0.0,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -406,11 +406,11 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
@@ -426,12 +426,12 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
-                "accuracy": 0.0,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -446,12 +446,12 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.0,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.0,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.0,
+                "score_ci_high": 0.0,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -465,248 +465,248 @@
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
-            "score": 0.24489795918367346,
+            "score": 0.19387755102040816,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.2682539682539683,
+                "f1_macro": 0.13333333333333333,
                 "f1_suggestive": 0.16666666666666666,
-                "f1_fanciful": 0.3333333333333333,
-                "f1_generic": 0.2222222222222222,
-                "f1_descriptive": 0.3333333333333333,
-                "f1_arbitrary": 0.2857142857142857,
-                "f1_macro_ci_low": 0.12857142857142856,
-                "f1_macro_ci_high": 0.5753151207818901,
+                "f1_arbitrary": 0.5,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.0,
+                "f1_descriptive": 0.0,
+                "f1_macro_ci_low": 0.029544504197655135,
+                "f1_macro_ci_high": 0.275,
                 "score_name": "f1_micro",
-                "score": 0.25,
-                "score_ci_high": 0.5,
-                "score_ci_low": 0.1,
+                "score": 0.15,
+                "score_ci_high": 0.37270394126013257,
+                "score_ci_low": 0.05,
                 "num_of_instances": 20,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.25,
-                "f1_micro_ci_low": 0.1,
-                "f1_micro_ci_high": 0.5
+                "accuracy": 0.15,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.3672085770953458,
+                "f1_micro": 0.15,
+                "f1_micro_ci_low": 0.05,
+                "f1_micro_ci_high": 0.37270394126013257
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.5238095238095238,
-                "f1_no": 0.7142857142857143,
-                "f1_yes": 0.3333333333333333,
-                "f1_macro_ci_low": 0.3103448275862069,
-                "f1_macro_ci_high": 0.7619047619047619,
+                "f1_macro": 0.41333333333333333,
+                "f1_no": 0.56,
+                "f1_yes": 0.26666666666666666,
+                "f1_macro_ci_low": 0.23273657289002558,
+                "f1_macro_ci_high": 0.6865203761755485,
                 "score_name": "f1_micro",
-                "score": 0.6,
-                "score_ci_high": 0.8,
-                "score_ci_low": 0.35,
+                "score": 0.45,
+                "score_ci_high": 0.6666666666666666,
+                "score_ci_low": 0.25,
                 "num_of_instances": 20,
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.35,
-                "accuracy_ci_high": 0.8,
-                "f1_micro": 0.6,
-                "f1_micro_ci_low": 0.35,
-                "f1_micro_ci_high": 0.8
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.6549800691648727,
+                "f1_micro": 0.45,
+                "f1_micro_ci_low": 0.25,
+                "f1_micro_ci_high": 0.6666666666666666
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.06168831168831169,
-                "f1_conclusion": 0.25,
-                "f1_analysis": 0.18181818181818182,
+                "f1_macro": 0.05372405372405372,
+                "f1_conclusion": 0.2222222222222222,
+                "f1_analysis": 0.15384615384615385,
                 "f1_decree": 0.0,
-                "f1_issue": 0.0,
                 "f1_facts": 0.0,
+                "f1_issue": 0.0,
                 "f1_rule": 0.0,
                 "f1_procedural history": 0.0,
                 "f1_macro_ci_low": 0.0,
-                "f1_macro_ci_high": 0.19169137881749854,
+                "f1_macro_ci_high": 0.15349110987889103,
                 "score_name": "f1_micro",
-                "score": 0.10810810810810811,
-                "score_ci_high": 0.32753517177309893,
+                "score": 0.1,
+                "score_ci_high": 0.3,
                 "score_ci_low": 0.0,
                 "num_of_instances": 20,
                 "accuracy": 0.1,
                 "accuracy_ci_low": 0.0,
                 "accuracy_ci_high": 0.35,
-                "f1_micro": 0.10810810810810811,
+                "f1_micro": 0.1,
                 "f1_micro_ci_low": 0.0,
-                "f1_micro_ci_high": 0.32753517177309893
+                "f1_micro_ci_high": 0.3
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5059523809523809,
+                "f1_macro": 0.4916666666666667,
                 "f1_yes": 0.5833333333333334,
-                "f1_no": 0.42857142857142855,
-                "f1_macro_ci_low": 0.2916666666666667,
-                "f1_macro_ci_high": 0.7685099295204324,
+                "f1_no": 0.4,
+                "f1_macro_ci_low": 0.29994839408816376,
+                "f1_macro_ci_high": 0.7184210526315788,
                 "score_name": "f1_micro",
-                "score": 0.5263157894736842,
+                "score": 0.5128205128205128,
                 "score_ci_high": 0.7368421052631579,
                 "score_ci_low": 0.3076923076923077,
                 "num_of_instances": 20,
                 "accuracy": 0.5,
                 "accuracy_ci_low": 0.3,
                 "accuracy_ci_high": 0.7,
-                "f1_micro": 0.5263157894736842,
+                "f1_micro": 0.5128205128205128,
                 "f1_micro_ci_low": 0.3076923076923077,
                 "f1_micro_ci_high": 0.7368421052631579
             },
             "legalbench_proa": {
-                "f1_macro": 0.7777777777777778,
-                "f1_yes": 0.7777777777777778,
-                "f1_no": 0.7777777777777778,
-                "f1_macro_ci_low": 0.5645704686649926,
-                "f1_macro_ci_high": 0.9206349206349206,
+                "f1_macro": 0.6491228070175439,
+                "f1_yes": 0.631578947368421,
+                "f1_no": 0.6666666666666666,
+                "f1_macro_ci_low": 0.44862155388471175,
+                "f1_macro_ci_high": 0.8470120517129166,
                 "score_name": "f1_micro",
-                "score": 0.7777777777777778,
-                "score_ci_high": 0.918918918918919,
-                "score_ci_low": 0.5502143713513362,
+                "score": 0.65,
+                "score_ci_high": 0.8,
+                "score_ci_low": 0.4,
                 "num_of_instances": 20,
-                "accuracy": 0.7,
-                "accuracy_ci_low": 0.45,
-                "accuracy_ci_high": 0.85,
-                "f1_micro": 0.7777777777777778,
-                "f1_micro_ci_low": 0.5502143713513362,
-                "f1_micro_ci_high": 0.918918918918919
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.65,
+                "f1_micro_ci_low": 0.4,
+                "f1_micro_ci_high": 0.8
             },
-            "score": 0.452440335071914,
+            "score": 0.37256410256410255,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.10357142857142856,
-                "f1_cars": 0.5,
+                "f1_macro": 0.13032051282051282,
+                "f1_cars": 0.6,
                 "f1_windows x": 0.0,
                 "f1_atheism": 0.0,
+                "f1_politics": 0.3076923076923077,
                 "f1_religion": 0.0,
                 "f1_medicine": 0.0,
-                "f1_hockey": 0.0,
                 "f1_christianity": 0.0,
-                "f1_computer graphics": 0.0,
+                "f1_computer graphics": 0.3333333333333333,
                 "f1_microsoft windows": 0.0,
                 "f1_middle east": 0.0,
                 "f1_motorcycles": 0.0,
-                "f1_cryptography": 0.0,
                 "f1_mac hardware": 0.0,
                 "f1_electronics": 0.0,
                 "f1_for sale": 0.0,
                 "f1_guns": 0.0,
-                "f1_politics": 0.25,
-                "f1_space": 0.5714285714285714,
+                "f1_space": 0.75,
                 "f1_pc hardware": 0.0,
-                "f1_baseball": 0.75,
-                "f1_macro_ci_low": 0.06018184125447124,
-                "f1_macro_ci_high": 0.14696961511980738,
+                "f1_cryptography": 0.0,
+                "f1_baseball": 0.6153846153846154,
+                "f1_hockey": 0.0,
+                "f1_macro_ci_low": 0.08300993096063736,
+                "f1_macro_ci_high": 0.17568685492942918,
                 "score_name": "f1_micro",
-                "score": 0.16783216783216784,
-                "score_ci_high": 0.2698797944115196,
-                "score_ci_low": 0.0881843925888208,
+                "score": 0.2,
+                "score_ci_high": 0.3120567375886525,
+                "score_ci_low": 0.11678832116788321,
                 "num_of_instances": 100,
-                "accuracy": 0.12,
-                "accuracy_ci_low": 0.07,
-                "accuracy_ci_high": 0.2,
-                "f1_micro": 0.16783216783216784,
-                "f1_micro_ci_low": 0.0881843925888208,
-                "f1_micro_ci_high": 0.2698797944115196
+                "accuracy": 0.14,
+                "accuracy_ci_low": 0.08,
+                "accuracy_ci_high": 0.23,
+                "f1_micro": 0.2,
+                "f1_micro_ci_low": 0.11678832116788321,
+                "f1_micro_ci_high": 0.3120567375886525
             },
-            "score": 0.16783216783216784,
+            "score": 0.2,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.33580801337810684,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.5981308411214953,
-                "f1_money transfer or virtual currency or money service": 0.0,
+                "f1_macro": 0.3019138755980861,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.631578947368421,
                 "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_money transfer or virtual currency or money service": 0.0,
+                "f1_checking or savings account": 0.18181818181818182,
                 "f1_mortgage": 1.0,
-                "f1_credit card or prepaid card": 0.16666666666666666,
-                "f1_checking or savings account": 0.36363636363636365,
-                "f1_debt collection": 0.2222222222222222,
-                "f1_macro_ci_low": 0.194694481168003,
-                "f1_macro_ci_high": 0.5251343899790525,
+                "f1_credit card or prepaid card": 0.0,
+                "f1_debt collection": 0.3,
+                "f1_macro_ci_low": 0.17209384713135212,
+                "f1_macro_ci_high": 0.4611549930931899,
                 "score_name": "f1_micro",
-                "score": 0.49032258064516127,
-                "score_ci_high": 0.6010548560611424,
-                "score_ci_low": 0.3901548550180111,
+                "score": 0.5125,
+                "score_ci_high": 0.6128968486960344,
+                "score_ci_low": 0.40808931189367126,
                 "num_of_instances": 100,
-                "accuracy": 0.38,
-                "accuracy_ci_low": 0.29,
-                "accuracy_ci_high": 0.49,
-                "f1_micro": 0.49032258064516127,
-                "f1_micro_ci_low": 0.3901548550180111,
-                "f1_micro_ci_high": 0.6010548560611424
+                "accuracy": 0.41,
+                "accuracy_ci_low": 0.32,
+                "accuracy_ci_high": 0.5131183635822549,
+                "f1_micro": 0.5125,
+                "f1_micro_ci_low": 0.40808931189367126,
+                "f1_micro_ci_high": 0.6128968486960344
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.40016806722689074,
-                "f1_mortgages and loans": 0.25,
-                "f1_debt collection": 0.5294117647058824,
-                "f1_credit card": 0.4,
-                "f1_credit reporting": 0.5714285714285714,
-                "f1_retail banking": 0.25,
-                "f1_macro_ci_low": 0.2813509457307339,
-                "f1_macro_ci_high": 0.5595889234269605,
+                "f1_macro": 0.31565591397849463,
+                "f1_mortgages and loans": 0.4,
+                "f1_debt collection": 0.45161290322580644,
+                "f1_credit card": 0.16666666666666666,
+                "f1_credit reporting": 0.56,
+                "f1_retail banking": 0.0,
+                "f1_macro_ci_low": 0.21637195045262772,
+                "f1_macro_ci_high": 0.4421317990640253,
                 "score_name": "f1_micro",
-                "score": 0.44680851063829785,
-                "score_ci_high": 0.5894736842105263,
-                "score_ci_low": 0.3164301824285703,
+                "score": 0.4,
+                "score_ci_high": 0.5319148936170213,
+                "score_ci_low": 0.2608695652173913,
                 "num_of_instances": 50,
-                "accuracy": 0.42,
-                "accuracy_ci_low": 0.3,
-                "accuracy_ci_high": 0.56,
-                "f1_micro": 0.44680851063829785,
-                "f1_micro_ci_low": 0.3164301824285703,
-                "f1_micro_ci_high": 0.5894736842105263
+                "accuracy": 0.38,
+                "accuracy_ci_low": 0.24,
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.4,
+                "f1_micro_ci_low": 0.2608695652173913,
+                "f1_micro_ci_high": 0.5319148936170213
             },
-            "score": 0.4685655456417296,
+            "score": 0.45625,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.06,
-                "score": 0.06,
+                "program_accuracy": 0.07,
+                "score": 0.07,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.04,
-                "program_accuracy_ci_low": 0.02,
-                "program_accuracy_ci_high": 0.12,
-                "score_ci_low": 0.02,
-                "score_ci_high": 0.12,
-                "execution_accuracy_ci_low": 0.01,
-                "execution_accuracy_ci_high": 0.1
+                "execution_accuracy": 0.05,
+                "program_accuracy_ci_low": 0.03,
+                "program_accuracy_ci_high": 0.13,
+                "score_ci_low": 0.03,
+                "score_ci_high": 0.13,
+                "execution_accuracy_ci_low": 0.02,
+                "execution_accuracy_ci_high": 0.11
             },
-            "score": 0.06,
+            "score": 0.07,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.5394200704985423,
-                "recall": 0.4471996521738687,
-                "f1": 0.4374006616033173,
-                "precision_ci_low": 0.49576534801510874,
-                "precision_ci_high": 0.5813735731767599,
-                "recall_ci_low": 0.4038227974845249,
-                "recall_ci_high": 0.4893013713568421,
-                "f1_ci_low": 0.4063022709069138,
-                "f1_ci_high": 0.47437539074022306,
+                "precision": 0.523939092007554,
+                "recall": 0.4630665384804083,
+                "f1": 0.4470066183026211,
+                "precision_ci_low": 0.48178772883210663,
+                "precision_ci_high": 0.5657398750195303,
+                "recall_ci_low": 0.4204989413298855,
+                "recall_ci_high": 0.5060265317751137,
+                "f1_ci_low": 0.40883623553494647,
+                "f1_ci_high": 0.48378981442802294,
                 "score_name": "f1",
-                "score": 0.4374006616033173,
-                "score_ci_high": 0.47437539074022306,
-                "score_ci_low": 0.4063022709069138,
+                "score": 0.4470066183026211,
+                "score_ci_high": 0.48378981442802294,
+                "score_ci_low": 0.40883623553494647,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6614969465136528,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6515003818273545,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6838387006521225,
-                "faithfullness_f1_token_overlap": 0.30369595667553545,
-                "faithfullness_recall_token_overlap": 0.22023180928573866,
-                "faithfullness_precision_token_overlap": 0.7308649191201634,
-                "correctness_f1_token_overlap": 0.4374006616033173,
-                "correctness_recall_token_overlap": 0.4471996521738687,
-                "correctness_precision_token_overlap": 0.5394200704985423
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6648982459306717,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.661893335878849,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6803490483760833,
+                "faithfullness_f1_token_overlap": 0.3107343678104924,
+                "faithfullness_recall_token_overlap": 0.2175296397330215,
+                "faithfullness_precision_token_overlap": 0.7439144424456543,
+                "correctness_f1_token_overlap": 0.4470066183026211,
+                "correctness_recall_token_overlap": 0.4630665384804083,
+                "correctness_precision_token_overlap": 0.523939092007554
             },
-            "score": 0.4374006616033173,
+            "score": 0.4470066183026211,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -722,85 +722,85 @@
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.56,
-                "accuracy_ci_low": 0.46,
-                "accuracy_ci_high": 0.65,
+                "accuracy": 0.54,
+                "accuracy_ci_low": 0.44,
+                "accuracy_ci_high": 0.63,
                 "score_name": "accuracy",
-                "score": 0.56,
-                "score_ci_high": 0.65,
-                "score_ci_low": 0.46,
+                "score": 0.54,
+                "score_ci_high": 0.63,
+                "score_ci_low": 0.44,
                 "num_of_instances": 100
             },
-            "score": 0.42000000000000004,
+            "score": 0.41000000000000003,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
         "safety": {
             "attaq_500": {
                 "severity_critical": 0.0,
-                "severity_high": 2.0,
+                "severity_high": 0.0,
                 "severity_medium": 1.0,
-                "severity_low": 97.0,
-                "category_harmful_info": 0.8452391722519043,
-                "category_deception": 0.8451698155640954,
-                "category_explicit_content": 0.8289112692905796,
-                "category_violence": 0.8006358703805342,
-                "category_discrimination": 0.7370405632352072,
-                "category_substance_abuse": 0.8171433870991072,
-                "category_pii": 0.8536775884804902,
-                "safety": 0.8190987786335688,
-                "safety_ci_low": 0.7945383739210665,
-                "safety_ci_high": 0.8410869711974197,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8276903443038464,
+                "category_deception": 0.8509014063712325,
+                "category_explicit_content": 0.8699462725733867,
+                "category_violence": 0.823169431019397,
+                "category_discrimination": 0.768092628982332,
+                "category_substance_abuse": 0.8343986455765036,
+                "category_pii": 0.8457972087241985,
+                "safety": 0.8300161745150882,
+                "safety_ci_low": 0.8130536746996235,
+                "safety_ci_high": 0.847588766122457,
                 "score_name": "safety",
-                "score": 0.8190987786335688,
-                "score_ci_high": 0.8410869711974197,
-                "score_ci_low": 0.7945383739210665,
+                "score": 0.8300161745150882,
+                "score_ci_high": 0.847588766122457,
+                "score_ci_low": 0.8130536746996235,
                 "num_of_instances": 100
             },
-            "score": 0.8190987786335688,
+            "score": 0.8300161745150882,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.2726363210739404,
-                "score": 0.2726363210739404,
+                "rougeLsum": 0.3432004189089268,
+                "rouge1": 0.4042025000472494,
+                "rougeL": 0.27446864632702683,
+                "score": 0.27446864632702683,
                 "score_name": "rougeL",
-                "rouge2": 0.18608701571308153,
-                "rouge1": 0.4022994438648122,
-                "rougeLsum": 0.34361980166371475,
-                "rougeL_ci_low": 0.2551498105781197,
-                "rougeL_ci_high": 0.2899026771230966,
-                "score_ci_low": 0.2551498105781197,
-                "score_ci_high": 0.2899026771230966,
-                "rouge2_ci_low": 0.168814962712534,
-                "rouge2_ci_high": 0.20356774988182394,
-                "rouge1_ci_low": 0.3782875736721082,
-                "rouge1_ci_high": 0.42469320666459026,
-                "rougeLsum_ci_low": 0.3218158445558661,
-                "rougeLsum_ci_high": 0.3641162818529719
+                "rouge2": 0.19586805105606808,
+                "rougeLsum_ci_low": 0.3220913706590708,
+                "rougeLsum_ci_high": 0.3618832326344797,
+                "rouge1_ci_low": 0.37966598950175057,
+                "rouge1_ci_high": 0.42455036578796235,
+                "rougeL_ci_low": 0.25908686634534905,
+                "rougeL_ci_high": 0.2912289617368165,
+                "score_ci_low": 0.25908686634534905,
+                "score_ci_high": 0.2912289617368165,
+                "rouge2_ci_low": 0.1802587963974639,
+                "rouge2_ci_high": 0.21175333730941023
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.07538049696363007,
-                "score": 0.07538049696363007,
+                "rougeLsum": 0.08407246092877058,
+                "rouge1": 0.10193639708473491,
+                "rougeL": 0.07786051266730082,
+                "score": 0.07786051266730082,
                 "score_name": "rougeL",
-                "rouge2": 0.012771223902408348,
-                "rouge1": 0.10098952560279839,
-                "rougeLsum": 0.08485681708183587,
-                "rougeL_ci_low": 0.06453651407342219,
-                "rougeL_ci_high": 0.086339345681574,
-                "score_ci_low": 0.06453651407342219,
-                "score_ci_high": 0.086339345681574,
-                "rouge2_ci_low": 0.008152944394513806,
-                "rouge2_ci_high": 0.02001103015890974,
-                "rouge1_ci_low": 0.08535457370632381,
-                "rouge1_ci_high": 0.11647938663113463,
-                "rougeLsum_ci_low": 0.07240452773193602,
-                "rougeLsum_ci_high": 0.09677432801226872
+                "rouge2": 0.011991811023543525,
+                "rougeLsum_ci_low": 0.07261612496065988,
+                "rougeLsum_ci_high": 0.09607861850289276,
+                "rouge1_ci_low": 0.08699540238464068,
+                "rouge1_ci_high": 0.11680045934783663,
+                "rougeL_ci_low": 0.067976352668634,
+                "rougeL_ci_high": 0.08831833326675104,
+                "score_ci_low": 0.067976352668634,
+                "score_ci_high": 0.08831833326675104,
+                "rouge2_ci_low": 0.008358797906991256,
+                "rouge2_ci_high": 0.017262915294486415
             },
-            "score": 0.17400840901878525,
+            "score": 0.17616457949716383,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    134,
-                    81,
-                    55,
-                    37
+                    116,
+                    60,
+                    35,
+                    19
                 ],
                 "totals": [
-                    217,
-                    211,
-                    205,
-                    199
+                    199,
+                    193,
+                    187,
+                    181
                 ],
                 "precisions": [
-                    0.6175115207373272,
-                    0.38388625592417064,
-                    0.2682926829268293,
-                    0.18592964824120603
+                    0.5829145728643216,
+                    0.31088082901554404,
+                    0.1871657754010695,
+                    0.10497237569060774
                 ],
-                "bp": 1.0,
-                "sys_len": 217,
+                "bp": 0.9557813259386698,
+                "sys_len": 199,
                 "ref_len": 208,
-                "sacrebleu": 0.32976250692588743,
-                "score": 0.32976250692588743,
+                "sacrebleu": 0.2334713639801202,
+                "score": 0.2334713639801202,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1930375611696375,
-                "score_ci_high": 0.45542436819873006,
-                "sacrebleu_ci_low": 0.1930375611696375,
-                "sacrebleu_ci_high": 0.45542436819873006
+                "score_ci_low": 0.15943718765537457,
+                "score_ci_high": 0.31107836291057467,
+                "sacrebleu_ci_low": 0.15943718765537457,
+                "sacrebleu_ci_high": 0.31107836291057467
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    120,
-                    60,
-                    30,
-                    16
+                    116,
+                    59,
+                    31,
+                    18
                 ],
                 "totals": [
-                    205,
-                    199,
-                    193,
-                    187
+                    191,
+                    185,
+                    179,
+                    173
                 ],
                 "precisions": [
-                    0.5853658536585367,
-                    0.30150753768844224,
-                    0.15544041450777202,
-                    0.08556149732620322
+                    0.6073298429319371,
+                    0.31891891891891894,
+                    0.17318435754189945,
+                    0.10404624277456648
                 ],
-                "bp": 0.9854724123463497,
-                "sys_len": 205,
+                "bp": 0.9148407838195897,
+                "sys_len": 191,
                 "ref_len": 208,
-                "sacrebleu": 0.21691342969872396,
-                "score": 0.21691342969872396,
+                "sacrebleu": 0.22235940836247658,
+                "score": 0.22235940836247658,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.12320730361123032,
-                "score_ci_high": 0.3487349414946175,
-                "sacrebleu_ci_low": 0.12320730361123032,
-                "sacrebleu_ci_high": 0.3487349414946175
+                "score_ci_low": 0.12250090520971424,
+                "score_ci_high": 0.39386620331090566,
+                "sacrebleu_ci_low": 0.12250090520971424,
+                "sacrebleu_ci_high": 0.39386620331090566
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    60,
-                    15,
-                    6,
-                    2
+                    62,
+                    12,
+                    3,
+                    1
                 ],
                 "totals": [
-                    511,
-                    505,
-                    499,
-                    493
+                    211,
+                    205,
+                    199,
+                    193
                 ],
                 "precisions": [
-                    0.11741682974559688,
-                    0.0297029702970297,
-                    0.012024048096192386,
-                    0.004056795131845842
+                    0.2938388625592417,
+                    0.058536585365853655,
+                    0.015075376884422112,
+                    0.005181347150259067
                 ],
                 "bp": 1.0,
-                "sys_len": 511,
+                "sys_len": 211,
                 "ref_len": 209,
-                "sacrebleu": 0.020309115200914007,
-                "score": 0.020309115200914007,
+                "sacrebleu": 0.03404566896908617,
+                "score": 0.03404566896908617,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0031235807690743827,
-                "score_ci_high": 0.08413507865164595,
-                "sacrebleu_ci_low": 0.0031235807690743827,
-                "sacrebleu_ci_high": 0.08413507865164595
+                "score_ci_low": 0.014238102093315122,
+                "score_ci_high": 0.06777961671960171,
+                "sacrebleu_ci_low": 0.014238102093315122,
+                "sacrebleu_ci_high": 0.06777961671960171
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    110,
-                    55,
-                    32,
-                    22
+                    104,
+                    47,
+                    26,
+                    13
                 ],
                 "totals": [
-                    204,
-                    198,
-                    192,
-                    186
+                    224,
+                    218,
+                    212,
+                    206
                 ],
                 "precisions": [
-                    0.5392156862745098,
-                    0.2777777777777778,
-                    0.16666666666666669,
-                    0.11827956989247312
+                    0.4642857142857143,
+                    0.21559633027522934,
+                    0.12264150943396226,
+                    0.06310679611650485
                 ],
-                "bp": 0.9428731438548749,
-                "sys_len": 204,
+                "bp": 1.0,
+                "sys_len": 224,
                 "ref_len": 216,
-                "sacrebleu": 0.2197899810468473,
-                "score": 0.2197899810468473,
+                "sacrebleu": 0.1668341972740045,
+                "score": 0.1668341972740045,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.07840487865437475,
-                "score_ci_high": 0.36940461297883537,
-                "sacrebleu_ci_low": 0.07840487865437475,
-                "sacrebleu_ci_high": 0.36940461297883537
+                "score_ci_low": 0.08494307414866337,
+                "score_ci_high": 0.2669068447403921,
+                "sacrebleu_ci_low": 0.08494307414866337,
+                "sacrebleu_ci_high": 0.2669068447403921
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    153,
-                    83,
-                    48,
-                    28
+                    157,
+                    95,
+                    64,
+                    43
                 ],
                 "totals": [
-                    231,
-                    225,
-                    219,
-                    213
+                    249,
+                    243,
+                    237,
+                    231
                 ],
                 "precisions": [
-                    0.6623376623376623,
-                    0.3688888888888889,
-                    0.2191780821917808,
-                    0.13145539906103287
+                    0.6305220883534137,
+                    0.39094650205761317,
+                    0.27004219409282704,
+                    0.18614718614718614
                 ],
-                "bp": 0.9828330432930387,
-                "sys_len": 231,
+                "bp": 1.0,
+                "sys_len": 249,
                 "ref_len": 235,
-                "sacrebleu": 0.28468684086858825,
-                "score": 0.28468684086858825,
+                "sacrebleu": 0.3336387113992972,
+                "score": 0.3336387113992972,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2199495343644957,
-                "score_ci_high": 0.3534794457690317,
-                "sacrebleu_ci_low": 0.2199495343644957,
-                "sacrebleu_ci_high": 0.3534794457690317
+                "score_ci_low": 0.21817943053845332,
+                "score_ci_high": 0.41307713856625455,
+                "sacrebleu_ci_low": 0.21817943053845332,
+                "sacrebleu_ci_high": 0.41307713856625455
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    104,
-                    40,
-                    20,
-                    11
+                    97,
+                    35,
+                    13,
+                    6
                 ],
                 "totals": [
-                    274,
-                    268,
-                    262,
-                    256
+                    1750,
+                    1744,
+                    1738,
+                    1732
                 ],
                 "precisions": [
-                    0.3795620437956205,
-                    0.1492537313432836,
-                    0.07633587786259542,
-                    0.04296875
+                    0.055428571428571424,
+                    0.02006880733944954,
+                    0.007479861910241657,
+                    0.003464203233256351
                 ],
                 "bp": 1.0,
-                "sys_len": 274,
+                "sys_len": 1750,
                 "ref_len": 249,
-                "sacrebleu": 0.11675415620470718,
-                "score": 0.11675415620470718,
+                "sacrebleu": 0.013029808954660122,
+                "score": 0.013029808954660122,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06337490011995338,
-                "score_ci_high": 0.1718798128824584,
-                "sacrebleu_ci_low": 0.06337490011995338,
-                "sacrebleu_ci_high": 0.1718798128824584
+                "score_ci_low": 0.005042950798954314,
+                "score_ci_high": 0.08046612284400166,
+                "sacrebleu_ci_low": 0.005042950798954314,
+                "sacrebleu_ci_high": 0.08046612284400166
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    160,
-                    110,
-                    83,
-                    63
+                    158,
+                    108,
+                    79,
+                    58
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    213,
+                    207,
+                    201,
+                    195
                 ],
                 "precisions": [
-                    0.7582938388625593,
-                    0.5365853658536586,
-                    0.41708542713567837,
-                    0.3264248704663213
+                    0.7417840375586855,
+                    0.5217391304347826,
+                    0.3930348258706468,
+                    0.29743589743589743
                 ],
-                "bp": 0.9492028979108159,
-                "sys_len": 211,
+                "bp": 0.9586267176373937,
+                "sys_len": 213,
                 "ref_len": 222,
-                "sacrebleu": 0.46050060744945276,
-                "score": 0.46050060744945276,
+                "sacrebleu": 0.4421181624666089,
+                "score": 0.4421181624666089,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3827306936268306,
-                "score_ci_high": 0.5671546178089313,
-                "sacrebleu_ci_low": 0.3827306936268306,
-                "sacrebleu_ci_high": 0.5671546178089313
+                "score_ci_low": 0.36288486848367396,
+                "score_ci_high": 0.5136032567408664,
+                "sacrebleu_ci_low": 0.36288486848367396,
+                "sacrebleu_ci_high": 0.5136032567408664
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    115,
-                    60,
-                    38,
-                    24
+                    123,
+                    63,
+                    41,
+                    32
                 ],
                 "totals": [
-                    229,
-                    223,
-                    217,
-                    211
+                    227,
+                    221,
+                    215,
+                    209
                 ],
                 "precisions": [
-                    0.5021834061135371,
-                    0.26905829596412556,
-                    0.17511520737327188,
-                    0.1137440758293839
+                    0.5418502202643172,
+                    0.28506787330316746,
+                    0.19069767441860463,
+                    0.15311004784688995
                 ],
-                "bp": 0.9956427084340843,
-                "sys_len": 229,
+                "bp": 0.9868710869905453,
+                "sys_len": 227,
                 "ref_len": 230,
-                "sacrebleu": 0.2267743162542805,
-                "score": 0.2267743162542805,
+                "sacrebleu": 0.25574348566082084,
+                "score": 0.25574348566082084,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.15815899648450668,
-                "score_ci_high": 0.35001467739505476,
-                "sacrebleu_ci_low": 0.15815899648450668,
-                "sacrebleu_ci_high": 0.35001467739505476
+                "score_ci_low": 0.13165649076081204,
+                "score_ci_high": 0.4446310754662533,
+                "sacrebleu_ci_low": 0.13165649076081204,
+                "sacrebleu_ci_high": 0.4446310754662533
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    140,
-                    76,
-                    45,
-                    29
+                    141,
+                    80,
+                    48,
+                    33
                 ],
                 "totals": [
-                    230,
-                    224,
-                    218,
-                    212
+                    223,
+                    217,
+                    211,
+                    205
                 ],
                 "precisions": [
-                    0.6086956521739131,
-                    0.3392857142857143,
-                    0.20642201834862384,
-                    0.13679245283018868
+                    0.632286995515695,
+                    0.36866359447004604,
+                    0.2274881516587678,
+                    0.16097560975609757
                 ],
-                "bp": 0.9450459397948837,
-                "sys_len": 230,
+                "bp": 0.914218114531173,
+                "sys_len": 223,
                 "ref_len": 243,
-                "sacrebleu": 0.2611553356291334,
-                "score": 0.2611553356291334,
+                "sacrebleu": 0.2778853574267633,
+                "score": 0.2778853574267633,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.19269379023230448,
-                "score_ci_high": 0.32743500263915765,
-                "sacrebleu_ci_low": 0.19269379023230448,
-                "sacrebleu_ci_high": 0.32743500263915765
+                "score_ci_low": 0.18397606219224352,
+                "score_ci_high": 0.3891671906561486,
+                "sacrebleu_ci_low": 0.18397606219224352,
+                "sacrebleu_ci_high": 0.3891671906561486
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    150,
-                    86,
-                    53,
-                    31
+                    141,
+                    80,
+                    46,
+                    26
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    210,
+                    204,
+                    198,
+                    192
                 ],
                 "precisions": [
-                    0.7109004739336493,
-                    0.41951219512195126,
-                    0.2663316582914573,
-                    0.1606217616580311
+                    0.6714285714285714,
+                    0.3921568627450981,
+                    0.23232323232323232,
+                    0.13541666666666666
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 210,
                 "ref_len": 208,
-                "sacrebleu": 0.33608191487395417,
-                "score": 0.33608191487395417,
+                "sacrebleu": 0.3016866548631982,
+                "score": 0.3016866548631982,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24361347546013137,
-                "score_ci_high": 0.42191893112004974,
-                "sacrebleu_ci_low": 0.24361347546013137,
-                "sacrebleu_ci_high": 0.42191893112004974
+                "score_ci_low": 0.19135476382396907,
+                "score_ci_high": 0.41499391897561666,
+                "sacrebleu_ci_low": 0.19135476382396907,
+                "sacrebleu_ci_high": 0.41499391897561666
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    96,
-                    30,
-                    11,
-                    5
+                    114,
+                    43,
+                    19,
+                    9
                 ],
                 "totals": [
-                    184,
-                    178,
-                    172,
-                    166
+                    199,
+                    193,
+                    187,
+                    181
                 ],
                 "precisions": [
-                    0.5217391304347826,
-                    0.16853932584269665,
-                    0.06395348837209303,
-                    0.030120481927710843
+                    0.5728643216080402,
+                    0.2227979274611399,
+                    0.10160427807486631,
+                    0.04972375690607735
                 ],
-                "bp": 0.8777137332821824,
-                "sys_len": 184,
+                "bp": 0.9557813259386698,
+                "sys_len": 199,
                 "ref_len": 208,
-                "sacrebleu": 0.10013204777722429,
-                "score": 0.10013204777722429,
+                "sacrebleu": 0.15230643528886997,
+                "score": 0.15230643528886997,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06558487118000197,
-                "score_ci_high": 0.10887330550609021,
-                "sacrebleu_ci_low": 0.06558487118000197,
-                "sacrebleu_ci_high": 0.10887330550609021
+                "score_ci_low": 0.08366452659105056,
+                "score_ci_high": 0.25458346437891377,
+                "sacrebleu_ci_low": 0.08366452659105056,
+                "sacrebleu_ci_high": 0.25458346437891377
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    91,
-                    30,
-                    14,
-                    8
+                    84,
+                    24,
+                    10,
+                    5
                 ],
                 "totals": [
-                    188,
-                    182,
-                    176,
-                    170
+                    195,
+                    189,
+                    183,
+                    177
                 ],
                 "precisions": [
-                    0.48404255319148937,
-                    0.1648351648351648,
-                    0.07954545454545454,
-                    0.047058823529411764
+                    0.4307692307692308,
+                    0.12698412698412698,
+                    0.0546448087431694,
+                    0.028248587570621472
                 ],
-                "bp": 0.8990802535245078,
-                "sys_len": 188,
+                "bp": 0.9355069850316178,
+                "sys_len": 195,
                 "ref_len": 208,
-                "sacrebleu": 0.11819413324799515,
-                "score": 0.11819413324799515,
+                "sacrebleu": 0.0896771865535579,
+                "score": 0.0896771865535579,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.03870206520478138,
-                "score_ci_high": 0.2385085096019695,
-                "sacrebleu_ci_low": 0.03870206520478138,
-                "sacrebleu_ci_high": 0.2385085096019695
+                "score_ci_low": 0.026997714791803698,
+                "score_ci_high": 0.18938435699619116,
+                "sacrebleu_ci_low": 0.026997714791803698,
+                "sacrebleu_ci_high": 0.18938435699619116
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    141,
-                    87,
-                    58,
-                    39
+                    145,
+                    99,
+                    68,
+                    47
                 ],
                 "totals": [
-                    204,
-                    198,
-                    192,
-                    186
+                    200,
+                    194,
+                    188,
+                    182
                 ],
                 "precisions": [
-                    0.6911764705882354,
-                    0.4393939393939394,
-                    0.3020833333333333,
-                    0.20967741935483872
+                    0.725,
+                    0.5103092783505155,
+                    0.36170212765957444,
+                    0.25824175824175827
                 ],
-                "bp": 0.9805831403241088,
-                "sys_len": 204,
+                "bp": 0.9607894391523232,
+                "sys_len": 200,
                 "ref_len": 208,
-                "sacrebleu": 0.36518655576785464,
-                "score": 0.36518655576785464,
+                "sacrebleu": 0.4142528184241974,
+                "score": 0.4142528184241974,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2601914895570609,
-                "score_ci_high": 0.43326095285270216,
-                "sacrebleu_ci_low": 0.2601914895570609,
-                "sacrebleu_ci_high": 0.43326095285270216
+                "score_ci_low": 0.2751136761392381,
+                "score_ci_high": 0.5016871777154166,
+                "sacrebleu_ci_low": 0.2751136761392381,
+                "sacrebleu_ci_high": 0.5016871777154166
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    149,
-                    87,
+                    139,
+                    80,
                     53,
-                    34
+                    37
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    220,
+                    214,
+                    208,
+                    202
                 ],
                 "precisions": [
-                    0.6622222222222223,
-                    0.3972602739726028,
-                    0.2488262910798122,
-                    0.16425120772946858
+                    0.6318181818181818,
+                    0.37383177570093457,
+                    0.2548076923076923,
+                    0.18316831683168316
                 ],
                 "bp": 1.0,
-                "sys_len": 225,
+                "sys_len": 220,
                 "ref_len": 208,
-                "sacrebleu": 0.3220111659882407,
-                "score": 0.3220111659882407,
+                "sacrebleu": 0.32402819799793336,
+                "score": 0.32402819799793336,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2299064596013062,
-                "score_ci_high": 0.4146903200437098,
-                "sacrebleu_ci_low": 0.2299064596013062,
-                "sacrebleu_ci_high": 0.4146903200437098
+                "score_ci_low": 0.17933889414478565,
+                "score_ci_high": 0.4618316777407589,
+                "sacrebleu_ci_low": 0.17933889414478565,
+                "sacrebleu_ci_high": 0.4618316777407589
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    125,
-                    73,
-                    42,
-                    26
+                    118,
+                    59,
+                    33,
+                    20
                 ],
                 "totals": [
-                    206,
-                    200,
-                    194,
-                    188
+                    207,
+                    201,
+                    195,
+                    189
                 ],
                 "precisions": [
-                    0.6067961165048543,
-                    0.365,
-                    0.21649484536082475,
-                    0.13829787234042554
+                    0.5700483091787439,
+                    0.2935323383084577,
+                    0.16923076923076924,
+                    0.10582010582010583
                 ],
-                "bp": 0.9903382397772544,
-                "sys_len": 206,
+                "bp": 0.9951807322415573,
+                "sys_len": 207,
                 "ref_len": 208,
-                "sacrebleu": 0.28260720363612,
-                "score": 0.28260720363612,
+                "sacrebleu": 0.23283900945772085,
+                "score": 0.23283900945772085,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1851284935850412,
-                "score_ci_high": 0.39347136307788017,
-                "sacrebleu_ci_low": 0.1851284935850412,
-                "sacrebleu_ci_high": 0.39347136307788017
+                "score_ci_low": 0.1202707272049383,
+                "score_ci_high": 0.3401632328329729,
+                "sacrebleu_ci_low": 0.1202707272049383,
+                "sacrebleu_ci_high": 0.3401632328329729
             },
-            "score": 0.24405728737132826,
+            "score": 0.23292776447195437,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.32787857652155017,
+        "score": 0.3182745669848931,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T16-23-36_evaluation_results.json → 2025-07-03T10-08-21_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T20:23:32.663416Z",
+        "timestamp_utc": "2025-07-03T14:08:17.472494Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,36 +176,46 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.41707199293005626,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.41707199293005626,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
             "safety_bbq_disability_status": {
-                "accuracy": 0.4444444444444444,
-                "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.7777777777777778,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.4444444444444444,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.1111111111111111,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.7777777777777778,
+                "accuracy": 0.6666666666666666,
                 "accuracy_ci_low": 0.3333333333333333,
-                "accuracy_ci_high": 1.0,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 1.0,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
                 "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
             "safety_bbq_nationality": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
+                "score_name": "accuracy",
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
                 "accuracy": 0.5555555555555556,
                 "accuracy_ci_low": 0.2222222222222222,
                 "accuracy_ci_high": 0.8888888888888888,
@@ -215,27 +225,27 @@
                 "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
-            "safety_bbq_physical_appearance": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.41707199293005626,
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5310928992288233,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.41707199293005626,
+                "score_ci_low": 0.5310928992288233,
                 "num_of_instances": 9
             },
-            "safety_bbq_race_ethnicity": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
-                "accuracy_ci_high": 1.0,
+            "safety_bbq_race_x_gender": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
                 "score_name": "accuracy",
-                "score": 1.0,
-                "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
                 "num_of_instances": 9
             },
-            "safety_bbq_race_x_gender": {
+            "safety_bbq_race_x_ses": {
                 "accuracy": 0.6666666666666666,
                 "accuracy_ci_low": 0.3333333333333333,
                 "accuracy_ci_high": 0.8888888888888888,
@@ -245,16 +255,6 @@
                 "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
-            "safety_bbq_race_x_ses": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.4444444444444444,
-                "accuracy_ci_high": 1.0,
-                "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.4444444444444444,
-                "num_of_instances": 9
-            },
             "safety_bbq_religion": {
                 "accuracy": 0.6666666666666666,
                 "accuracy_ci_low": 0.3333333333333333,
@@ -266,73 +266,73 @@
                 "num_of_instances": 9
             },
             "safety_bbq_ses": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.6666666666666666,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.0,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.6666666666666666,
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.0,
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
                 "num_of_instances": 9
             },
-            "score": 0.6464646464646464,
+            "score": 0.6060606060606061,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.2777777777777778,
-                "score": 0.2777777777777778,
+                "llama_3_70b_instruct_template_arena_hard": 0.25139664804469275,
+                "score": 0.25139664804469275,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.2777777777777778,
+            "score": 0.25139664804469275,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.33333333333333337,
-                "f1_Organization": 0.16326530612244897,
-                "f1_Location": 0.29411764705882354,
-                "f1_macro": 0.2635720955048686,
-                "recall_macro": 0.2040200138026225,
-                "precision_macro": 0.384004884004884,
-                "in_classes_support": 0.7719298245614035,
-                "f1_micro": 0.22727272727272727,
-                "recall_micro": 0.2,
-                "precision_micro": 0.2631578947368421,
-                "score": 0.22727272727272727,
+                "f1_Person": 0.5,
+                "f1_Organization": 0.5084745762711865,
+                "f1_Location": 0.34146341463414637,
+                "f1_macro": 0.4499793303017776,
+                "recall_macro": 0.4062284334023465,
+                "precision_macro": 0.5293144553106602,
+                "in_classes_support": 0.9838709677419355,
+                "f1_micro": 0.45255474452554745,
+                "recall_micro": 0.41333333333333333,
+                "precision_micro": 0.5,
+                "score": 0.45255474452554745,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.13940062602106976,
-                "score_ci_high": 0.3364095639750836,
-                "f1_micro_ci_low": 0.13940062602106976,
-                "f1_micro_ci_high": 0.3364095639750836
+                "score_ci_low": 0.3227264456627763,
+                "score_ci_high": 0.5928025470401035,
+                "f1_micro_ci_low": 0.3227264456627763,
+                "f1_micro_ci_high": 0.5928025470401035
             },
-            "score": 0.22727272727272727,
+            "score": 0.45255474452554745,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.5714285714285714,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
@@ -346,23 +346,23 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_economics": {
@@ -376,22 +376,22 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.0,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -406,32 +406,32 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
-                "accuracy": 0.2857142857142857,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -446,12 +446,12 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -465,38 +465,38 @@
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
-            "score": 0.2857142857142857,
+            "score": 0.35714285714285715,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.21875901875901876,
-                "f1_suggestive": 0.0,
-                "f1_descriptive": 0.36363636363636365,
+                "f1_macro": 0.4031746031746032,
+                "f1_suggestive": 0.3333333333333333,
                 "f1_generic": 0.0,
-                "f1_fanciful": 0.2857142857142857,
-                "f1_arbitrary": 0.4444444444444444,
-                "f1_macro_ci_low": 0.1,
-                "f1_macro_ci_high": 0.4116972927189486,
+                "f1_descriptive": 0.4444444444444444,
+                "f1_fanciful": 0.5714285714285714,
+                "f1_arbitrary": 0.6666666666666666,
+                "f1_macro_ci_low": 0.2350094328437234,
+                "f1_macro_ci_high": 0.6281441125339357,
                 "score_name": "f1_micro",
-                "score": 0.25,
-                "score_ci_high": 0.45,
-                "score_ci_low": 0.1,
+                "score": 0.45,
+                "score_ci_high": 0.65,
+                "score_ci_low": 0.25,
                 "num_of_instances": 20,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.45,
-                "f1_micro": 0.25,
-                "f1_micro_ci_low": 0.1,
-                "f1_micro_ci_high": 0.45
+                "accuracy": 0.45,
+                "accuracy_ci_low": 0.25,
+                "accuracy_ci_high": 0.65,
+                "f1_micro": 0.45,
+                "f1_micro_ci_low": 0.25,
+                "f1_micro_ci_high": 0.65
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.4373401534526854,
-                "f1_no": 0.35294117647058826,
-                "f1_yes": 0.5217391304347826,
+                "f1_macro": 0.44862155388471175,
+                "f1_no": 0.47619047619047616,
+                "f1_yes": 0.42105263157894735,
                 "f1_macro_ci_low": 0.24812030075187969,
-                "f1_macro_ci_high": 0.6493608471738732,
+                "f1_macro_ci_high": 0.696969696969697,
                 "score_name": "f1_micro",
                 "score": 0.45,
                 "score_ci_high": 0.65,
@@ -510,228 +510,228 @@
                 "f1_micro_ci_high": 0.65
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.32173382173382176,
-                "f1_conclusion": 0.3076923076923077,
-                "f1_analysis": 0.0,
+                "f1_macro": 0.3128220985363842,
+                "f1_conclusion": 0.18181818181818182,
+                "f1_analysis": 0.2222222222222222,
                 "f1_decree": 0.0,
-                "f1_issue": 0.4444444444444444,
+                "f1_issue": 0.2857142857142857,
                 "f1_facts": 0.5,
                 "f1_procedural history": 1.0,
                 "f1_rule": 0.0,
-                "f1_macro_ci_low": 0.16826244282941694,
-                "f1_macro_ci_high": 0.5700272265278143,
+                "f1_macro_ci_low": 0.16666666666666666,
+                "f1_macro_ci_high": 0.5627298511545029,
                 "score_name": "f1_micro",
-                "score": 0.3076923076923077,
-                "score_ci_high": 0.5294117647058824,
-                "score_ci_low": 0.10526315789473684,
+                "score": 0.2631578947368421,
+                "score_ci_high": 0.4864864864864865,
+                "score_ci_low": 0.10256410256410256,
                 "num_of_instances": 20,
-                "accuracy": 0.3,
-                "accuracy_ci_low": 0.15,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.3076923076923077,
-                "f1_micro_ci_low": 0.10526315789473684,
-                "f1_micro_ci_high": 0.5294117647058824
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.49471586405580864,
+                "f1_micro": 0.2631578947368421,
+                "f1_micro_ci_low": 0.10256410256410256,
+                "f1_micro_ci_high": 0.4864864864864865
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5833333333333333,
-                "f1_yes": 0.6666666666666666,
-                "f1_no": 0.5,
-                "f1_macro_ci_low": 0.3483709273182957,
-                "f1_macro_ci_high": 0.797979797979798,
+                "f1_macro": 0.595959595959596,
+                "f1_yes": 0.6363636363636364,
+                "f1_no": 0.5555555555555556,
+                "f1_macro_ci_low": 0.3732193732193732,
+                "f1_macro_ci_high": 0.8,
                 "score_name": "f1_micro",
                 "score": 0.6,
                 "score_ci_high": 0.8,
-                "score_ci_low": 0.4,
+                "score_ci_low": 0.35,
                 "num_of_instances": 20,
                 "accuracy": 0.6,
-                "accuracy_ci_low": 0.4,
+                "accuracy_ci_low": 0.35,
                 "accuracy_ci_high": 0.8,
                 "f1_micro": 0.6,
-                "f1_micro_ci_low": 0.4,
+                "f1_micro_ci_low": 0.35,
                 "f1_micro_ci_high": 0.8
             },
             "legalbench_proa": {
-                "f1_macro": 0.7222222222222222,
-                "f1_yes": 0.6666666666666666,
+                "f1_macro": 0.7638888888888888,
+                "f1_yes": 0.75,
                 "f1_no": 0.7777777777777778,
-                "f1_macro_ci_low": 0.4917908886027495,
-                "f1_macro_ci_high": 0.8742358224473542,
+                "f1_macro_ci_low": 0.5298844011348174,
+                "f1_macro_ci_high": 0.8985941651727045,
                 "score_name": "f1_micro",
-                "score": 0.7272727272727273,
-                "score_ci_high": 0.8648648648648649,
-                "score_ci_low": 0.5078970996299597,
+                "score": 0.7647058823529411,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.5454545454545454,
                 "num_of_instances": 20,
-                "accuracy": 0.6,
+                "accuracy": 0.65,
                 "accuracy_ci_low": 0.4,
                 "accuracy_ci_high": 0.8,
-                "f1_micro": 0.7272727272727273,
-                "f1_micro_ci_low": 0.5078970996299597,
-                "f1_micro_ci_high": 0.8648648648648649
+                "f1_micro": 0.7647058823529411,
+                "f1_micro_ci_low": 0.5454545454545454,
+                "f1_micro_ci_high": 0.8888888888888888
             },
-            "score": 0.466993006993007,
+            "score": 0.5055727554179567,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.2824164578111946,
+                "f1_macro": 0.2910756911460236,
                 "f1_cars": 0.6,
                 "f1_windows x": 0.0,
                 "f1_atheism": 0.0,
                 "f1_christianity": 0.0,
                 "f1_religion": 0.0,
                 "f1_medicine": 0.3333333333333333,
-                "f1_computer graphics": 0.3157894736842105,
-                "f1_microsoft windows": 0.4444444444444444,
+                "f1_computer graphics": 0.2608695652173913,
+                "f1_microsoft windows": 0.6,
                 "f1_middle east": 0.0,
-                "f1_politics": 0.5714285714285714,
-                "f1_motorcycles": 0.0,
+                "f1_politics": 0.6666666666666666,
+                "f1_motorcycles": 0.25,
                 "f1_mac hardware": 0.3333333333333333,
-                "f1_pc hardware": 0.3333333333333333,
+                "f1_pc hardware": 0.5,
                 "f1_for sale": 0.0,
+                "f1_electronics": 0.5,
                 "f1_guns": 0.5,
-                "f1_space": 0.75,
+                "f1_baseball": 0.7058823529411765,
+                "f1_space": 0.5714285714285714,
                 "f1_cryptography": 0.0,
-                "f1_baseball": 0.8,
-                "f1_electronics": 0.6666666666666666,
                 "f1_hockey": 0.0,
-                "f1_macro_ci_low": 0.22475419677111075,
-                "f1_macro_ci_high": 0.36930934309468255,
+                "f1_macro_ci_low": 0.22542343425145048,
+                "f1_macro_ci_high": 0.3817631536924331,
                 "score_name": "f1_micro",
-                "score": 0.34782608695652173,
-                "score_ci_high": 0.45707499587676786,
-                "score_ci_low": 0.25313614072293233,
+                "score": 0.36470588235294116,
+                "score_ci_high": 0.4678362573099415,
+                "score_ci_low": 0.26400794870774236,
                 "num_of_instances": 100,
-                "accuracy": 0.28,
-                "accuracy_ci_low": 0.2,
-                "accuracy_ci_high": 0.38,
-                "f1_micro": 0.34782608695652173,
-                "f1_micro_ci_low": 0.25313614072293233,
-                "f1_micro_ci_high": 0.45707499587676786
+                "accuracy": 0.31,
+                "accuracy_ci_low": 0.22,
+                "accuracy_ci_high": 0.41,
+                "f1_micro": 0.36470588235294116,
+                "f1_micro_ci_low": 0.26400794870774236,
+                "f1_micro_ci_high": 0.4678362573099415
             },
-            "score": 0.34782608695652173,
+            "score": 0.36470588235294116,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.5681318681318681,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9,
-                "f1_credit card or prepaid card": 0.3076923076923077,
+                "f1_macro": 0.7523191638915732,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9130434782608695,
+                "f1_credit card or prepaid card": 0.2857142857142857,
                 "f1_money transfer or virtual currency or money service": 0.6666666666666666,
                 "f1_mortgage": 1.0,
-                "f1_debt collection": 0.3333333333333333,
+                "f1_debt collection": 0.631578947368421,
                 "f1_checking or savings account": 0.7692307692307693,
-                "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_macro_ci_low": 0.3709228365796495,
-                "f1_macro_ci_high": 0.744785019590881,
+                "f1_payday loan or title loan or personal loan": 1.0,
+                "f1_macro_ci_low": 0.6489086297233181,
+                "f1_macro_ci_high": 0.9210195820841641,
                 "score_name": "f1_micro",
-                "score": 0.7835051546391752,
-                "score_ci_high": 0.8585858585858586,
-                "score_ci_low": 0.6975697603592736,
+                "score": 0.8247422680412371,
+                "score_ci_high": 0.8923076923076924,
+                "score_ci_low": 0.743197883853095,
                 "num_of_instances": 100,
-                "accuracy": 0.76,
-                "accuracy_ci_low": 0.67,
-                "accuracy_ci_high": 0.84,
-                "f1_micro": 0.7835051546391752,
-                "f1_micro_ci_low": 0.6975697603592736,
-                "f1_micro_ci_high": 0.8585858585858586
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.71,
+                "accuracy_ci_high": 0.87,
+                "f1_micro": 0.8247422680412371,
+                "f1_micro_ci_low": 0.743197883853095,
+                "f1_micro_ci_high": 0.8923076923076924
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.5947033358798065,
-                "f1_mortgages and loans": 0.7619047619047619,
-                "f1_credit card": 0.5454545454545454,
-                "f1_debt collection": 0.5882352941176471,
-                "f1_retail banking": 0.36363636363636365,
-                "f1_credit reporting": 0.7142857142857143,
-                "f1_macro_ci_low": 0.4651735405033739,
-                "f1_macro_ci_high": 0.7569013016004565,
+                "f1_macro": 0.6613469059121233,
+                "f1_mortgages and loans": 0.8695652173913043,
+                "f1_credit card": 0.5714285714285714,
+                "f1_debt collection": 0.625,
+                "f1_retail banking": 0.5,
+                "f1_credit reporting": 0.7407407407407407,
+                "f1_macro_ci_low": 0.5279754605261872,
+                "f1_macro_ci_high": 0.8082982300291591,
                 "score_name": "f1_micro",
-                "score": 0.6262626262626263,
-                "score_ci_high": 0.7676767676767676,
-                "score_ci_low": 0.4897959183673469,
+                "score": 0.6868686868686869,
+                "score_ci_high": 0.8163265306122449,
+                "score_ci_low": 0.54,
                 "num_of_instances": 50,
-                "accuracy": 0.62,
-                "accuracy_ci_low": 0.48,
-                "accuracy_ci_high": 0.76,
-                "f1_micro": 0.6262626262626263,
-                "f1_micro_ci_low": 0.4897959183673469,
-                "f1_micro_ci_high": 0.7676767676767676
+                "accuracy": 0.68,
+                "accuracy_ci_low": 0.54,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.6868686868686869,
+                "f1_micro_ci_low": 0.54,
+                "f1_micro_ci_high": 0.8163265306122449
             },
-            "score": 0.7048838904509007,
+            "score": 0.755805477454962,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.13,
-                "score": 0.13,
+                "program_accuracy": 0.1,
+                "score": 0.1,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.1,
-                "program_accuracy_ci_low": 0.07,
-                "program_accuracy_ci_high": 0.2,
-                "score_ci_low": 0.07,
-                "score_ci_high": 0.2,
-                "execution_accuracy_ci_low": 0.05,
-                "execution_accuracy_ci_high": 0.17
+                "execution_accuracy": 0.09,
+                "program_accuracy_ci_low": 0.05,
+                "program_accuracy_ci_high": 0.17,
+                "score_ci_low": 0.05,
+                "score_ci_high": 0.17,
+                "execution_accuracy_ci_low": 0.04,
+                "execution_accuracy_ci_high": 0.16
             },
-            "score": 0.13,
+            "score": 0.1,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.5259718170827713,
-                "recall": 0.5325565848189471,
-                "f1": 0.48788698516603257,
-                "precision_ci_low": 0.4911250761135513,
-                "precision_ci_high": 0.5653352686639593,
-                "recall_ci_low": 0.49293364282057184,
-                "recall_ci_high": 0.5717848110862319,
-                "f1_ci_low": 0.45660847819815814,
-                "f1_ci_high": 0.5189210923819673,
+                "precision": 0.5282260960979881,
+                "recall": 0.518990758836778,
+                "f1": 0.4781146759785926,
+                "precision_ci_low": 0.4905036545891592,
+                "precision_ci_high": 0.5712847375973066,
+                "recall_ci_low": 0.4792082705307863,
+                "recall_ci_high": 0.5643322796851563,
+                "f1_ci_low": 0.4464661091001953,
+                "f1_ci_high": 0.5127460974737836,
                 "score_name": "f1",
-                "score": 0.48788698516603257,
-                "score_ci_high": 0.5189210923819673,
-                "score_ci_low": 0.45660847819815814,
+                "score": 0.4781146759785926,
+                "score_ci_high": 0.5127460974737836,
+                "score_ci_low": 0.4464661091001953,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6809317979216576,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.691152431666851,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6823357805609703,
-                "faithfullness_f1_token_overlap": 0.34443287111054743,
-                "faithfullness_recall_token_overlap": 0.24725245575960428,
-                "faithfullness_precision_token_overlap": 0.7209445115448376,
-                "correctness_f1_token_overlap": 0.48788698516603257,
-                "correctness_recall_token_overlap": 0.5325565848189471,
-                "correctness_precision_token_overlap": 0.5259718170827713
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6747818207740783,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.683358971774578,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.678565673828125,
+                "faithfullness_f1_token_overlap": 0.3328173876076052,
+                "faithfullness_recall_token_overlap": 0.23833647131370628,
+                "faithfullness_precision_token_overlap": 0.7221546643746082,
+                "correctness_f1_token_overlap": 0.4781146759785926,
+                "correctness_recall_token_overlap": 0.518990758836778,
+                "correctness_precision_token_overlap": 0.5282260960979881
             },
-            "score": 0.48788698516603257,
+            "score": 0.4781146759785926,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.44,
-                "accuracy_ci_low": 0.34,
-                "accuracy_ci_high": 0.54,
+                "accuracy": 0.43,
+                "accuracy_ci_low": 0.33,
+                "accuracy_ci_high": 0.52,
                 "score_name": "accuracy",
-                "score": 0.44,
-                "score_ci_high": 0.54,
-                "score_ci_low": 0.34,
+                "score": 0.43,
+                "score_ci_high": 0.52,
+                "score_ci_low": 0.33,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.7,
+                "accuracy": 0.69,
                 "accuracy_ci_low": 0.6,
-                "accuracy_ci_high": 0.78,
+                "accuracy_ci_high": 0.77,
                 "score_name": "accuracy",
-                "score": 0.7,
-                "score_ci_high": 0.78,
+                "score": 0.69,
+                "score_ci_high": 0.77,
                 "score_ci_low": 0.6,
                 "num_of_instances": 100
             },
-            "score": 0.57,
+            "score": 0.5599999999999999,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -741,66 +741,66 @@
                 "severity_high": 0.0,
                 "severity_medium": 0.0,
                 "severity_low": 100.0,
-                "category_harmful_info": 0.846627970521767,
-                "category_deception": 0.8117720823396336,
-                "category_explicit_content": 0.8644592114771256,
-                "category_violence": 0.8120507185894345,
-                "category_discrimination": 0.8723783436275664,
-                "category_substance_abuse": 0.8963490577207672,
-                "category_pii": 0.7459356255001491,
-                "safety": 0.8339210312772125,
-                "safety_ci_low": 0.8157590553137859,
-                "safety_ci_high": 0.8500795039523255,
+                "category_harmful_info": 0.8555861621695972,
+                "category_deception": 0.8145618479360234,
+                "category_explicit_content": 0.8516158367497753,
+                "category_violence": 0.8195424805617049,
+                "category_discrimination": 0.7993254798671439,
+                "category_substance_abuse": 0.8629376106792026,
+                "category_pii": 0.8319368116833545,
+                "safety": 0.8337413073946824,
+                "safety_ci_low": 0.8171390500734657,
+                "safety_ci_high": 0.8483087143425718,
                 "score_name": "safety",
-                "score": 0.8339210312772125,
-                "score_ci_high": 0.8500795039523255,
-                "score_ci_low": 0.8157590553137859,
+                "score": 0.8337413073946824,
+                "score_ci_high": 0.8483087143425718,
+                "score_ci_low": 0.8171390500734657,
                 "num_of_instances": 100
             },
-            "score": 0.8339210312772125,
+            "score": 0.8337413073946824,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.4201269465168249,
-                "rougeLsum": 0.35887081152360367,
-                "rougeL": 0.2899376436388642,
-                "score": 0.2899376436388642,
+                "rougeLsum": 0.35257144277181895,
+                "rouge1": 0.4099938802688273,
+                "rougeL": 0.2819180609443961,
+                "score": 0.2819180609443961,
                 "score_name": "rougeL",
-                "rouge2": 0.2040248319542606,
-                "rouge1_ci_low": 0.3957950378754922,
-                "rouge1_ci_high": 0.44198037514196525,
-                "rougeLsum_ci_low": 0.33566125462866964,
-                "rougeLsum_ci_high": 0.37945926824008097,
-                "rougeL_ci_low": 0.27073181817971537,
-                "rougeL_ci_high": 0.3107229983596245,
-                "score_ci_low": 0.27073181817971537,
-                "score_ci_high": 0.3107229983596245,
-                "rouge2_ci_low": 0.18748753359560238,
-                "rouge2_ci_high": 0.2273071634840196
+                "rouge2": 0.19902668429160264,
+                "rougeLsum_ci_low": 0.3308328413239909,
+                "rougeLsum_ci_high": 0.3760952815354367,
+                "rouge1_ci_low": 0.38577957980842964,
+                "rouge1_ci_high": 0.43278596944262226,
+                "rougeL_ci_low": 0.2631055438860382,
+                "rougeL_ci_high": 0.3045234232476088,
+                "score_ci_low": 0.2631055438860382,
+                "score_ci_high": 0.3045234232476088,
+                "rouge2_ci_low": 0.18104502049358925,
+                "rouge2_ci_high": 0.2197355387706385
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.11018773543004212,
-                "rougeLsum": 0.08916779934450281,
-                "rougeL": 0.08311133323979714,
-                "score": 0.08311133323979714,
+                "rougeLsum": 0.08735851266090196,
+                "rouge1": 0.10541346314519143,
+                "rougeL": 0.07789268452336062,
+                "score": 0.07789268452336062,
                 "score_name": "rougeL",
-                "rouge2": 0.015505336966737249,
-                "rouge1_ci_low": 0.09524241123987982,
-                "rouge1_ci_high": 0.12446415160426098,
-                "rougeLsum_ci_low": 0.07806656289344661,
-                "rougeLsum_ci_high": 0.10055148752407524,
-                "rougeL_ci_low": 0.0727724269814928,
-                "rougeL_ci_high": 0.0928135162226291,
-                "score_ci_low": 0.0727724269814928,
-                "score_ci_high": 0.0928135162226291,
-                "rouge2_ci_low": 0.011076119889397521,
-                "rouge2_ci_high": 0.020887950896596442
+                "rouge2": 0.014214026654707938,
+                "rougeLsum_ci_low": 0.07611511674805775,
+                "rougeLsum_ci_high": 0.09861302754891539,
+                "rouge1_ci_low": 0.09141522803497244,
+                "rouge1_ci_high": 0.12087153728069493,
+                "rougeL_ci_low": 0.06842881641572805,
+                "rougeL_ci_high": 0.08804076692156412,
+                "score_ci_low": 0.06842881641572805,
+                "score_ci_high": 0.08804076692156412,
+                "rouge2_ci_low": 0.010280478508026637,
+                "rouge2_ci_high": 0.019836643868933537
             },
-            "score": 0.18652448843933067,
+            "score": 0.17990537273387835,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,196 +808,196 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    134,
-                    89,
-                    66,
-                    54
+                    138,
+                    86,
+                    60,
+                    41
                 ],
                 "totals": [
+                    217,
                     211,
                     205,
-                    199,
-                    193
+                    199
                 ],
                 "precisions": [
-                    0.6350710900473934,
-                    0.43414634146341463,
-                    0.3316582914572864,
-                    0.27979274611398963
+                    0.6359447004608295,
+                    0.4075829383886256,
+                    0.29268292682926833,
+                    0.20603015075376885
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 217,
                 "ref_len": 208,
-                "sacrebleu": 0.3999414802337733,
-                "score": 0.3999414802337733,
+                "sacrebleu": 0.35358259555851773,
+                "score": 0.35358259555851773,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09825253161374167,
-                "score_ci_high": 0.5276086387431496,
-                "sacrebleu_ci_low": 0.09825253161374167,
-                "sacrebleu_ci_high": 0.5276086387431496
+                "score_ci_low": 0.1947162858319295,
+                "score_ci_high": 0.5123065246583021,
+                "sacrebleu_ci_low": 0.1947162858319295,
+                "sacrebleu_ci_high": 0.5123065246583021
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    135,
-                    76,
-                    45,
+                    138,
+                    75,
+                    46,
                     31
                 ],
                 "totals": [
-                    212,
-                    206,
-                    200,
-                    194
+                    214,
+                    208,
+                    202,
+                    196
                 ],
                 "precisions": [
-                    0.6367924528301887,
-                    0.36893203883495146,
-                    0.225,
-                    0.15979381443298968
+                    0.6448598130841121,
+                    0.3605769230769231,
+                    0.2277227722772277,
+                    0.15816326530612243
                 ],
                 "bp": 1.0,
-                "sys_len": 212,
+                "sys_len": 214,
                 "ref_len": 208,
-                "sacrebleu": 0.30315985479098034,
-                "score": 0.30315985479098034,
+                "sacrebleu": 0.30251285307136316,
+                "score": 0.30251285307136316,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21741553813332964,
-                "score_ci_high": 0.44780469853287025,
-                "sacrebleu_ci_low": 0.21741553813332964,
-                "sacrebleu_ci_high": 0.44780469853287025
+                "score_ci_low": 0.19775244274276757,
+                "score_ci_high": 0.42106139492364764,
+                "sacrebleu_ci_low": 0.19775244274276757,
+                "sacrebleu_ci_high": 0.42106139492364764
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    71,
-                    26,
-                    13,
+                    83,
+                    30,
+                    14,
                     5
                 ],
                 "totals": [
-                    888,
-                    882,
-                    876,
-                    870
+                    199,
+                    193,
+                    187,
+                    181
                 ],
                 "precisions": [
-                    0.07995495495495496,
-                    0.02947845804988662,
-                    0.014840182648401827,
-                    0.005747126436781609
+                    0.41708542713567837,
+                    0.15544041450777202,
+                    0.0748663101604278,
+                    0.027624309392265192
                 ],
-                "bp": 1.0,
-                "sys_len": 888,
+                "bp": 0.9509904521556576,
+                "sys_len": 199,
                 "ref_len": 209,
-                "sacrebleu": 0.02117434748734448,
-                "score": 0.02117434748734448,
+                "sacrebleu": 0.10233350793678746,
+                "score": 0.10233350793678746,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0034363647304289997,
-                "score_ci_high": 0.12617952190760182,
-                "sacrebleu_ci_low": 0.0034363647304289997,
-                "sacrebleu_ci_high": 0.12617952190760182
+                "score_ci_low": 0.05050155487196502,
+                "score_ci_high": 0.17307034784752537,
+                "sacrebleu_ci_low": 0.05050155487196502,
+                "sacrebleu_ci_high": 0.17307034784752537
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    120,
-                    63,
-                    34,
-                    20
+                    125,
+                    67,
+                    39,
+                    25
                 ],
                 "totals": [
-                    221,
-                    215,
-                    209,
-                    203
+                    222,
+                    216,
+                    210,
+                    204
                 ],
                 "precisions": [
-                    0.5429864253393665,
-                    0.2930232558139535,
-                    0.1626794258373206,
-                    0.09852216748768473
+                    0.5630630630630631,
+                    0.3101851851851852,
+                    0.18571428571428572,
+                    0.12254901960784315
                 ],
                 "bp": 1.0,
-                "sys_len": 221,
+                "sys_len": 222,
                 "ref_len": 216,
-                "sacrebleu": 0.22471880288808055,
-                "score": 0.22471880288808055,
+                "sacrebleu": 0.25109225147784964,
+                "score": 0.25109225147784964,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10763204349992618,
-                "score_ci_high": 0.366138035767947,
-                "sacrebleu_ci_low": 0.10763204349992618,
-                "sacrebleu_ci_high": 0.366138035767947
+                "score_ci_low": 0.1529878549251057,
+                "score_ci_high": 0.3720906061784551,
+                "sacrebleu_ci_low": 0.1529878549251057,
+                "sacrebleu_ci_high": 0.3720906061784551
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    167,
-                    112,
-                    82,
-                    61
+                    160,
+                    103,
+                    72,
+                    50
                 ],
                 "totals": [
-                    236,
-                    230,
-                    224,
-                    218
+                    238,
+                    232,
+                    226,
+                    220
                 ],
                 "precisions": [
-                    0.7076271186440678,
-                    0.48695652173913045,
-                    0.36607142857142855,
-                    0.2798165137614679
+                    0.6722689075630253,
+                    0.4439655172413793,
+                    0.3185840707964602,
+                    0.22727272727272727
                 ],
                 "bp": 1.0,
-                "sys_len": 236,
+                "sys_len": 238,
                 "ref_len": 235,
-                "sacrebleu": 0.43344446111073914,
-                "score": 0.43344446111073914,
+                "sacrebleu": 0.38341218564513063,
+                "score": 0.38341218564513063,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3499428856979402,
-                "score_ci_high": 0.5041148126175684,
-                "sacrebleu_ci_low": 0.3499428856979402,
-                "sacrebleu_ci_high": 0.5041148126175684
+                "score_ci_low": 0.2731562527670829,
+                "score_ci_high": 0.4801134379288344,
+                "sacrebleu_ci_low": 0.2731562527670829,
+                "sacrebleu_ci_high": 0.4801134379288344
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    125,
-                    55,
-                    26,
-                    12
+                    136,
+                    52,
+                    25,
+                    15
                 ],
                 "totals": [
-                    294,
-                    288,
-                    282,
-                    276
+                    295,
+                    289,
+                    283,
+                    277
                 ],
                 "precisions": [
-                    0.4251700680272109,
-                    0.1909722222222222,
-                    0.0921985815602837,
-                    0.043478260869565216
+                    0.4610169491525424,
+                    0.17993079584775085,
+                    0.08833922261484099,
+                    0.05415162454873646
                 ],
                 "bp": 1.0,
-                "sys_len": 294,
+                "sys_len": 295,
                 "ref_len": 249,
-                "sacrebleu": 0.13431741406488118,
-                "score": 0.13431741406488118,
+                "sacrebleu": 0.14113894412300834,
+                "score": 0.14113894412300834,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08622044326635767,
-                "score_ci_high": 0.18006459720682508,
-                "sacrebleu_ci_low": 0.08622044326635767,
-                "sacrebleu_ci_high": 0.18006459720682508
+                "score_ci_low": 0.09931106147481285,
+                "score_ci_high": 0.21049896887236444,
+                "sacrebleu_ci_low": 0.09931106147481285,
+                "sacrebleu_ci_high": 0.21049896887236444
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    162,
-                    115,
-                    85,
-                    66
+                    164,
+                    122,
+                    95,
+                    75
                 ],
                 "totals": [
                     217,
@@ -1006,275 +1006,275 @@
                     199
                 ],
                 "precisions": [
-                    0.7465437788018433,
-                    0.5450236966824644,
-                    0.4146341463414634,
-                    0.3316582914572864
+                    0.7557603686635944,
+                    0.5781990521327014,
+                    0.4634146341463415,
+                    0.37688442211055273
                 ],
                 "bp": 0.977221952990032,
                 "sys_len": 217,
                 "ref_len": 222,
-                "sacrebleu": 0.47528035317523853,
-                "score": 0.47528035317523853,
+                "sacrebleu": 0.5136331789412277,
+                "score": 0.5136331789412277,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3904344196224883,
-                "score_ci_high": 0.6010867967504717,
-                "sacrebleu_ci_low": 0.3904344196224883,
-                "sacrebleu_ci_high": 0.6010867967504717
+                "score_ci_low": 0.4383895318184532,
+                "score_ci_high": 0.6687737636839535,
+                "sacrebleu_ci_low": 0.4383895318184532,
+                "sacrebleu_ci_high": 0.6687737636839535
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    141,
-                    87,
-                    63,
-                    46
+                    150,
+                    89,
+                    66,
+                    51
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    228,
+                    222,
+                    216,
+                    210
                 ],
                 "precisions": [
-                    0.6266666666666666,
-                    0.3972602739726028,
-                    0.29577464788732394,
-                    0.2222222222222222
+                    0.6578947368421052,
+                    0.4009009009009009,
+                    0.3055555555555556,
+                    0.24285714285714285
                 ],
-                "bp": 0.9780228724846006,
-                "sys_len": 225,
+                "bp": 0.9912664313028773,
+                "sys_len": 228,
                 "ref_len": 230,
-                "sacrebleu": 0.34979536672149464,
-                "score": 0.34979536672149464,
+                "sacrebleu": 0.3707652531456926,
+                "score": 0.3707652531456926,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.27052307609565535,
-                "score_ci_high": 0.46625375184661805,
-                "sacrebleu_ci_low": 0.27052307609565535,
-                "sacrebleu_ci_high": 0.46625375184661805
+                "score_ci_low": 0.24832285270330914,
+                "score_ci_high": 0.5155607169133788,
+                "sacrebleu_ci_low": 0.24832285270330914,
+                "sacrebleu_ci_high": 0.5155607169133788
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    87,
-                    51,
-                    31
+                    144,
+                    76,
+                    46,
+                    29
                 ],
                 "totals": [
-                    229,
-                    223,
-                    217,
-                    211
+                    221,
+                    215,
+                    209,
+                    203
                 ],
                 "precisions": [
-                    0.6593886462882097,
-                    0.3901345291479821,
-                    0.2350230414746544,
-                    0.14691943127962084
+                    0.6515837104072397,
+                    0.35348837209302325,
+                    0.22009569377990432,
+                    0.14285714285714288
                 ],
-                "bp": 0.9406958880448453,
-                "sys_len": 229,
+                "bp": 0.9052469393768031,
+                "sys_len": 221,
                 "ref_len": 243,
-                "sacrebleu": 0.2887923132416329,
-                "score": 0.2887923132416329,
+                "sacrebleu": 0.2640777306505383,
+                "score": 0.2640777306505383,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23190126496152094,
-                "score_ci_high": 0.35200649860482036,
-                "sacrebleu_ci_low": 0.23190126496152094,
-                "sacrebleu_ci_high": 0.35200649860482036
+                "score_ci_low": 0.1937710060893329,
+                "score_ci_high": 0.3011423563537887,
+                "sacrebleu_ci_low": 0.1937710060893329,
+                "sacrebleu_ci_high": 0.3011423563537887
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    157,
-                    105,
-                    75,
-                    53
+                    151,
+                    94,
+                    62,
+                    39
                 ],
                 "totals": [
-                    214,
-                    208,
-                    202,
-                    196
+                    212,
+                    206,
+                    200,
+                    194
                 ],
                 "precisions": [
-                    0.733644859813084,
-                    0.5048076923076923,
-                    0.3712871287128713,
-                    0.27040816326530615
+                    0.7122641509433962,
+                    0.4563106796116505,
+                    0.31,
+                    0.20103092783505155
                 ],
                 "bp": 1.0,
-                "sys_len": 214,
+                "sys_len": 212,
                 "ref_len": 208,
-                "sacrebleu": 0.43912219013856996,
-                "score": 0.43912219013856996,
+                "sacrebleu": 0.3772520189081424,
+                "score": 0.3772520189081424,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.35256330907180095,
-                "score_ci_high": 0.5295701336610964,
-                "sacrebleu_ci_low": 0.35256330907180095,
-                "sacrebleu_ci_high": 0.5295701336610964
+                "score_ci_low": 0.2635438049622767,
+                "score_ci_high": 0.504082453205032,
+                "sacrebleu_ci_low": 0.2635438049622767,
+                "sacrebleu_ci_high": 0.504082453205032
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    120,
-                    51,
-                    28,
-                    18
+                    128,
+                    70,
+                    43,
+                    29
                 ],
                 "totals": [
-                    218,
-                    212,
-                    206,
-                    200
+                    220,
+                    214,
+                    208,
+                    202
                 ],
                 "precisions": [
-                    0.5504587155963303,
-                    0.24056603773584906,
-                    0.13592233009708737,
-                    0.09
+                    0.5818181818181818,
+                    0.3271028037383178,
+                    0.20673076923076925,
+                    0.14356435643564358
                 ],
                 "bp": 1.0,
-                "sys_len": 218,
+                "sys_len": 220,
                 "ref_len": 208,
-                "sacrebleu": 0.20061947843312603,
-                "score": 0.20061947843312603,
+                "sacrebleu": 0.27414531356236105,
+                "score": 0.27414531356236105,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.07445839277243944,
-                "score_ci_high": 0.35371176032415624,
-                "sacrebleu_ci_low": 0.07445839277243944,
-                "sacrebleu_ci_high": 0.35371176032415624
+                "score_ci_low": 0.11294664927548931,
+                "score_ci_high": 0.40418288954231424,
+                "sacrebleu_ci_low": 0.11294664927548931,
+                "sacrebleu_ci_high": 0.40418288954231424
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    111,
-                    48,
-                    25,
-                    13
+                    113,
+                    58,
+                    29,
+                    16
                 ],
                 "totals": [
-                    193,
-                    187,
-                    181,
-                    175
+                    206,
+                    200,
+                    194,
+                    188
                 ],
                 "precisions": [
-                    0.5751295336787564,
-                    0.2566844919786096,
-                    0.13812154696132597,
-                    0.07428571428571429
+                    0.5485436893203883,
+                    0.29,
+                    0.14948453608247422,
+                    0.0851063829787234
                 ],
-                "bp": 0.9252232610888251,
-                "sys_len": 193,
+                "bp": 0.9903382397772544,
+                "sys_len": 206,
                 "ref_len": 208,
-                "sacrebleu": 0.18252802404200877,
-                "score": 0.18252802404200877,
+                "sacrebleu": 0.21005133898189757,
+                "score": 0.21005133898189757,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11632952815932861,
-                "score_ci_high": 0.2243103062098973,
-                "sacrebleu_ci_low": 0.11632952815932861,
-                "sacrebleu_ci_high": 0.2243103062098973
+                "score_ci_low": 0.12086953193591132,
+                "score_ci_high": 0.3598579039274656,
+                "sacrebleu_ci_low": 0.12086953193591132,
+                "sacrebleu_ci_high": 0.3598579039274656
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    155,
-                    99,
-                    69,
-                    52
+                    159,
+                    108,
+                    77,
+                    57
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    213,
+                    207,
+                    201,
+                    195
                 ],
                 "precisions": [
-                    0.7345971563981043,
-                    0.48292682926829267,
-                    0.34673366834170855,
-                    0.2694300518134715
+                    0.7464788732394366,
+                    0.5217391304347826,
+                    0.38308457711442784,
+                    0.2923076923076923
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 213,
                 "ref_len": 208,
-                "sacrebleu": 0.42667103009537916,
-                "score": 0.42667103009537916,
+                "sacrebleu": 0.4569844903443308,
+                "score": 0.4569844903443308,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23618053251632215,
-                "score_ci_high": 0.5738399910229026,
-                "sacrebleu_ci_low": 0.23618053251632215,
-                "sacrebleu_ci_high": 0.5738399910229026
+                "score_ci_low": 0.2921302619401382,
+                "score_ci_high": 0.6012686604638934,
+                "sacrebleu_ci_low": 0.2921302619401382,
+                "sacrebleu_ci_high": 0.6012686604638934
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    152,
-                    97,
-                    63,
-                    46
+                    145,
+                    86,
+                    52,
+                    34
                 ],
                 "totals": [
-                    224,
-                    218,
-                    212,
-                    206
+                    207,
+                    201,
+                    195,
+                    189
                 ],
                 "precisions": [
-                    0.6785714285714286,
-                    0.444954128440367,
-                    0.2971698113207547,
-                    0.2233009708737864
+                    0.7004830917874396,
+                    0.42786069651741293,
+                    0.26666666666666666,
+                    0.1798941798941799
                 ],
-                "bp": 1.0,
-                "sys_len": 224,
+                "bp": 0.9951807322415573,
+                "sys_len": 207,
                 "ref_len": 208,
-                "sacrebleu": 0.37622835832744195,
-                "score": 0.37622835832744195,
+                "sacrebleu": 0.34460647466230765,
+                "score": 0.34460647466230765,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18968604641672795,
-                "score_ci_high": 0.517042761677566,
-                "sacrebleu_ci_low": 0.18968604641672795,
-                "sacrebleu_ci_high": 0.517042761677566
+                "score_ci_low": 0.24134693897071277,
+                "score_ci_high": 0.477848957265904,
+                "sacrebleu_ci_low": 0.24134693897071277,
+                "sacrebleu_ci_high": 0.477848957265904
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    139,
-                    85,
+                    141,
+                    81,
                     49,
                     33
                 ],
                 "totals": [
-                    219,
-                    213,
-                    207,
-                    201
+                    223,
+                    217,
+                    211,
+                    205
                 ],
                 "precisions": [
-                    0.634703196347032,
-                    0.39906103286384975,
-                    0.23671497584541062,
-                    0.16417910447761194
+                    0.632286995515695,
+                    0.37327188940092165,
+                    0.23222748815165875,
+                    0.16097560975609757
                 ],
                 "bp": 1.0,
-                "sys_len": 219,
+                "sys_len": 223,
                 "ref_len": 208,
-                "sacrebleu": 0.31498393643128203,
-                "score": 0.31498393643128203,
+                "sacrebleu": 0.30648082606980787,
+                "score": 0.30648082606980787,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.22866378052841901,
-                "score_ci_high": 0.3789594362496447,
-                "sacrebleu_ci_low": 0.22866378052841901,
-                "sacrebleu_ci_high": 0.3789594362496447
+                "score_ci_low": 0.16929340790512817,
+                "score_ci_high": 0.3608174576081579,
+                "sacrebleu_ci_low": 0.16929340790512817,
+                "sacrebleu_ci_high": 0.3608174576081579
             },
-            "score": 0.3047184940787982,
+            "score": 0.31013793087193087,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.42076795543009543,
+        "score": 0.4427029429214344,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/2025-07-03T10-34-07_evaluation_results.json

@@ -0,0 +1,1281 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-07-03T14:34:02.551035Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/meta-llama/llama-3-2-90b-vision-instruct,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/meta-llama/llama-3-2-90b-vision-instruct",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.25.0",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "unitxt": "1.25.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "litellm": "1.73.6",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "tokenizers": "0.21.2",
+            "yarl": "1.20.1",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "openai": "1.93.0",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "pillow": "11.3.0",
+            "fonttools": "4.58.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "huggingface-hub": "0.33.2",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "accelerate": "1.8.1",
+            "certifi": "2025.6.15",
+            "evaluate": "0.4.4",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "transformers": "4.53.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "score": 1.0,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.8711656441717791,
+                "score": 0.8711656441717791,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.8711656441717791,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.8260869565217391,
+                "f1_Organization": 0.6551724137931035,
+                "f1_Location": 0.7272727272727272,
+                "f1_macro": 0.7361773658625234,
+                "recall_macro": 0.7237750172532781,
+                "precision_macro": 0.7531400966183576,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7297297297297296,
+                "recall_micro": 0.72,
+                "precision_micro": 0.7397260273972602,
+                "score": 0.7297297297297296,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.6619502313182618,
+                "score_ci_high": 0.7835819840150043,
+                "f1_micro_ci_low": 0.6619502313182618,
+                "f1_micro_ci_high": 0.7835819840150043
+            },
+            "score": 0.7297297297297296,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "score": 0.5510204081632653,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.8147619047619047,
+                "f1_suggestive": 0.6666666666666666,
+                "f1_generic": 1.0,
+                "f1_fanciful": 0.8571428571428571,
+                "f1_descriptive": 0.8,
+                "f1_arbitrary": 0.75,
+                "f1_macro_ci_low": 0.6241071521113625,
+                "f1_macro_ci_high": 0.9652136441488661,
+                "score_name": "f1_micro",
+                "score": 0.8,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.55,
+                "num_of_instances": 20,
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.55,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.8,
+                "f1_micro_ci_low": 0.55,
+                "f1_micro_ci_high": 0.95
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.7386363636363636,
+                "f1_no": 0.7272727272727273,
+                "f1_yes": 0.75,
+                "f1_macro_ci_low": 0.5080213903743316,
+                "f1_macro_ci_high": 0.9157902232720109,
+                "score_name": "f1_micro",
+                "score": 0.7368421052631579,
+                "score_ci_high": 0.8947368421052632,
+                "score_ci_low": 0.5128205128205128,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.5,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7368421052631579,
+                "f1_micro_ci_low": 0.5128205128205128,
+                "f1_micro_ci_high": 0.8947368421052632
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.17687074829931973,
+                "f1_conclusion": 0.2857142857142857,
+                "f1_decree": 0.0,
+                "f1_issue": 0.2857142857142857,
+                "f1_analysis": 0.6666666666666666,
+                "f1_facts": 0.0,
+                "f1_procedural history": 0.0,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.037037037037037035,
+                "f1_macro_ci_high": 0.3410139249890439,
+                "score_name": "f1_micro",
+                "score": 0.23529411764705882,
+                "score_ci_high": 0.48484848484848486,
+                "score_ci_low": 0.058823529411764705,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.45,
+                "f1_micro": 0.23529411764705882,
+                "f1_micro_ci_low": 0.058823529411764705,
+                "f1_micro_ci_high": 0.48484848484848486
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.6277777777777778,
+                "f1_yes": 0.7,
+                "f1_no": 0.5555555555555556,
+                "f1_macro_ci_low": 0.4143115659353126,
+                "f1_macro_ci_high": 0.849624060150376,
+                "score_name": "f1_micro",
+                "score": 0.631578947368421,
+                "score_ci_high": 0.8421052631578947,
+                "score_ci_low": 0.4069581788631691,
+                "num_of_instances": 20,
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.631578947368421,
+                "f1_micro_ci_low": 0.4069581788631691,
+                "f1_micro_ci_high": 0.8421052631578947
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.743421052631579,
+                "f1_yes": 0.75,
+                "f1_no": 0.7368421052631579,
+                "f1_macro_ci_low": 0.5133179285198034,
+                "f1_macro_ci_high": 0.898989898989899,
+                "score_name": "f1_micro",
+                "score": 0.7428571428571429,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.5,
+                "num_of_instances": 20,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.7428571428571429,
+                "f1_micro_ci_low": 0.5,
+                "f1_micro_ci_high": 0.8888888888888888
+            },
+            "score": 0.6293144626271561,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.6527228980170158,
+                "f1_cars": 1.0,
+                "f1_windows x": 0.5714285714285714,
+                "f1_computer graphics": 0.5882352941176471,
+                "f1_atheism": 0.3333333333333333,
+                "f1_christianity": 0.6666666666666666,
+                "f1_religion": 0.25,
+                "f1_medicine": 1.0,
+                "f1_microsoft windows": 0.8,
+                "f1_middle east": 0.5,
+                "f1_motorcycles": 0.7272727272727273,
+                "f1_pc hardware": 0.75,
+                "f1_mac hardware": 0.8888888888888888,
+                "f1_electronics": 0.5,
+                "f1_for sale": 0.8888888888888888,
+                "f1_guns": 0.4444444444444444,
+                "f1_space": 0.6,
+                "f1_cryptography": 0.3333333333333333,
+                "f1_baseball": 0.9230769230769231,
+                "f1_hockey": 0.8888888888888888,
+                "f1_politics": 0.4,
+                "f1_macro_ci_low": 0.5594466279416972,
+                "f1_macro_ci_high": 0.7484812715694672,
+                "score_name": "f1_micro",
+                "score": 0.6701030927835051,
+                "score_ci_high": 0.7525773195876289,
+                "score_ci_low": 0.5628781799105581,
+                "num_of_instances": 100,
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.54,
+                "accuracy_ci_high": 0.74,
+                "f1_micro": 0.6701030927835051,
+                "f1_micro_ci_low": 0.5628781799105581,
+                "f1_micro_ci_high": 0.7525773195876289
+            },
+            "score": 0.6701030927835051,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.7779168114934538,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9343065693430657,
+                "f1_credit card or prepaid card": 0.3333333333333333,
+                "f1_money transfer or virtual currency or money service": 0.8,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_debt collection": 0.7777777777777778,
+                "f1_checking or savings account": 0.9333333333333333,
+                "f1_payday loan or title loan or personal loan": 1.0,
+                "f1_macro_ci_low": 0.5932457464249341,
+                "f1_macro_ci_high": 0.8817165227032664,
+                "score_name": "f1_micro",
+                "score": 0.875,
+                "score_ci_high": 0.9238578680203046,
+                "score_ci_low": 0.8006847676679175,
+                "num_of_instances": 100,
+                "accuracy": 0.84,
+                "accuracy_ci_low": 0.76,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.875,
+                "f1_micro_ci_low": 0.8006847676679175,
+                "f1_micro_ci_high": 0.9238578680203046
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.8249444681938961,
+                "f1_mortgages and loans": 0.8695652173913043,
+                "f1_credit card": 0.7619047619047619,
+                "f1_debt collection": 0.7368421052631579,
+                "f1_credit reporting": 0.8333333333333334,
+                "f1_retail banking": 0.9230769230769231,
+                "f1_macro_ci_low": 0.6928894894256992,
+                "f1_macro_ci_high": 0.9177170275914066,
+                "score_name": "f1_micro",
+                "score": 0.82,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.68,
+                "num_of_instances": 50,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.68,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.82,
+                "f1_micro_ci_low": 0.68,
+                "f1_micro_ci_high": 0.9
+            },
+            "score": 0.8474999999999999,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "program_accuracy": 0.26,
+                "score": 0.26,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.25,
+                "program_accuracy_ci_low": 0.18,
+                "program_accuracy_ci_high": 0.35,
+                "score_ci_low": 0.18,
+                "score_ci_high": 0.35,
+                "execution_accuracy_ci_low": 0.17,
+                "execution_accuracy_ci_high": 0.34
+            },
+            "score": 0.26,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.5074323401711549,
+                "recall": 0.5740276560353169,
+                "f1": 0.49603980753298627,
+                "precision_ci_low": 0.47150664604306436,
+                "precision_ci_high": 0.5444306654624451,
+                "recall_ci_low": 0.5309832235253571,
+                "recall_ci_high": 0.6151274674565376,
+                "f1_ci_low": 0.46604824453263755,
+                "f1_ci_high": 0.5262523731070278,
+                "score_name": "f1",
+                "score": 0.49603980753298627,
+                "score_ci_high": 0.5262523731070278,
+                "score_ci_low": 0.46604824453263755,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6860781842470169,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.709225146472454,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6771670934557915,
+                "faithfullness_f1_token_overlap": 0.36312371853081976,
+                "faithfullness_recall_token_overlap": 0.2603375679120915,
+                "faithfullness_precision_token_overlap": 0.7208838371950508,
+                "correctness_f1_token_overlap": 0.49603980753298627,
+                "correctness_recall_token_overlap": 0.5740276560353169,
+                "correctness_precision_token_overlap": 0.5074323401711549
+            },
+            "score": 0.49603980753298627,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.57,
+                "accuracy_ci_low": 0.47,
+                "accuracy_ci_high": 0.66,
+                "score_name": "accuracy",
+                "score": 0.57,
+                "score_ci_high": 0.66,
+                "score_ci_low": 0.47,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.8636775645084782,
+                "accuracy_ci_high": 0.97,
+                "score_name": "accuracy",
+                "score": 0.93,
+                "score_ci_high": 0.97,
+                "score_ci_low": 0.8636775645084782,
+                "num_of_instances": 100
+            },
+            "score": 0.75,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 0.0,
+                "severity_high": 0.0,
+                "severity_medium": 1.0,
+                "severity_low": 99.0,
+                "category_harmful_info": 0.8584611813966482,
+                "category_deception": 0.884253122025367,
+                "category_explicit_content": 0.8562488650733774,
+                "category_violence": 0.8526820468745888,
+                "category_discrimination": 0.8608146858593775,
+                "category_substance_abuse": 0.8552402012877994,
+                "category_pii": 0.8454244594883035,
+                "safety": 0.8586448376372249,
+                "safety_ci_low": 0.8379978850647828,
+                "safety_ci_high": 0.8767468237261271,
+                "score_name": "safety",
+                "score": 0.8586448376372249,
+                "score_ci_high": 0.8767468237261271,
+                "score_ci_low": 0.8379978850647828,
+                "num_of_instances": 100
+            },
+            "score": 0.8586448376372249,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.20801889674008944,
+                "rougeL": 0.2968640964283658,
+                "score": 0.2968640964283658,
+                "score_name": "rougeL",
+                "rouge1": 0.42312199770919734,
+                "rougeLsum": 0.36721142345095514,
+                "rouge2_ci_low": 0.19303195128069772,
+                "rouge2_ci_high": 0.2239123862881902,
+                "rougeL_ci_low": 0.2793124666427084,
+                "rougeL_ci_high": 0.31605456043834845,
+                "score_ci_low": 0.2793124666427084,
+                "score_ci_high": 0.31605456043834845,
+                "rouge1_ci_low": 0.4001670264004444,
+                "rouge1_ci_high": 0.4447744439473787,
+                "rougeLsum_ci_low": 0.3471579822744466,
+                "rougeLsum_ci_high": 0.38772570475787616
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge2": 0.01604522130758397,
+                "rougeL": 0.08637910785624431,
+                "score": 0.08637910785624431,
+                "score_name": "rougeL",
+                "rouge1": 0.11342470214059448,
+                "rougeLsum": 0.09470632571584116,
+                "rouge2_ci_low": 0.010956366582946934,
+                "rouge2_ci_high": 0.02241153955079208,
+                "rougeL_ci_low": 0.07452990644153164,
+                "rougeL_ci_high": 0.09690240900042019,
+                "score_ci_low": 0.07452990644153164,
+                "score_ci_high": 0.09690240900042019,
+                "rouge1_ci_low": 0.09801484446894211,
+                "rouge1_ci_high": 0.12984514349711393,
+                "rougeLsum_ci_low": 0.08127416438123053,
+                "rougeLsum_ci_high": 0.10623331358688204
+            },
+            "score": 0.19162160214230506,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    161,
+                    119,
+                    91,
+                    71
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.7318181818181819,
+                    0.5560747663551402,
+                    0.4375,
+                    0.35148514851485146
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 208,
+                "sacrebleu": 0.500155852462094,
+                "score": 0.500155852462094,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.27107466732933977,
+                "score_ci_high": 0.6456509094349956,
+                "sacrebleu_ci_low": 0.27107466732933977,
+                "sacrebleu_ci_high": 0.6456509094349956
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    141,
+                    85,
+                    54,
+                    39
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.6527777777777777,
+                    0.40476190476190477,
+                    0.2647058823529412,
+                    0.19696969696969696
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.34259577311211076,
+                "score": 0.34259577311211076,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.22516597371165897,
+                "score_ci_high": 0.543023787423078,
+                "sacrebleu_ci_low": 0.22516597371165897,
+                "sacrebleu_ci_high": 0.543023787423078
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    131,
+                    80,
+                    50,
+                    29
+                ],
+                "totals": [
+                    203,
+                    197,
+                    191,
+                    185
+                ],
+                "precisions": [
+                    0.645320197044335,
+                    0.40609137055837563,
+                    0.2617801047120419,
+                    0.15675675675675677
+                ],
+                "bp": 0.9708758757257812,
+                "sys_len": 203,
+                "ref_len": 209,
+                "sacrebleu": 0.31264694630569706,
+                "score": 0.31264694630569706,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.21875352119497682,
+                "score_ci_high": 0.43847677077007524,
+                "sacrebleu_ci_low": 0.21875352119497682,
+                "sacrebleu_ci_high": 0.43847677077007524
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    144,
+                    94,
+                    68,
+                    52
+                ],
+                "totals": [
+                    224,
+                    218,
+                    212,
+                    206
+                ],
+                "precisions": [
+                    0.6428571428571429,
+                    0.4311926605504587,
+                    0.32075471698113206,
+                    0.2524271844660194
+                ],
+                "bp": 1.0,
+                "sys_len": 224,
+                "ref_len": 216,
+                "sacrebleu": 0.38705595372857227,
+                "score": 0.38705595372857227,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.270030091960412,
+                "score_ci_high": 0.5388939200476505,
+                "sacrebleu_ci_low": 0.270030091960412,
+                "sacrebleu_ci_high": 0.5388939200476505
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    188,
+                    150,
+                    122,
+                    100
+                ],
+                "totals": [
+                    244,
+                    238,
+                    232,
+                    226
+                ],
+                "precisions": [
+                    0.7704918032786885,
+                    0.6302521008403361,
+                    0.5258620689655172,
+                    0.4424778761061947
+                ],
+                "bp": 1.0,
+                "sys_len": 244,
+                "ref_len": 235,
+                "sacrebleu": 0.5797776009790664,
+                "score": 0.5797776009790664,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4819980312121314,
+                "score_ci_high": 0.7676936555305252,
+                "sacrebleu_ci_low": 0.4819980312121314,
+                "sacrebleu_ci_high": 0.7676936555305252
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    152,
+                    85,
+                    57,
+                    35
+                ],
+                "totals": [
+                    267,
+                    261,
+                    255,
+                    249
+                ],
+                "precisions": [
+                    0.5692883895131086,
+                    0.32567049808429116,
+                    0.22352941176470587,
+                    0.14056224899598393
+                ],
+                "bp": 1.0,
+                "sys_len": 267,
+                "ref_len": 249,
+                "sacrebleu": 0.27626669318098784,
+                "score": 0.27626669318098784,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.19630864275047086,
+                "score_ci_high": 0.3324774540569831,
+                "sacrebleu_ci_low": 0.19630864275047086,
+                "sacrebleu_ci_high": 0.3324774540569831
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    181,
+                    139,
+                    111,
+                    91
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.8008849557522124,
+                    0.6318181818181818,
+                    0.5186915887850467,
+                    0.4375
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 222,
+                "sacrebleu": 0.5821198107565924,
+                "score": 0.5821198107565924,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.5032683240695686,
+                "score_ci_high": 0.6631112459149506,
+                "sacrebleu_ci_low": 0.5032683240695686,
+                "sacrebleu_ci_high": 0.6631112459149506
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    160,
+                    108,
+                    80,
+                    62
+                ],
+                "totals": [
+                    233,
+                    227,
+                    221,
+                    215
+                ],
+                "precisions": [
+                    0.6866952789699571,
+                    0.47577092511013214,
+                    0.36199095022624433,
+                    0.28837209302325584
+                ],
+                "bp": 1.0,
+                "sys_len": 233,
+                "ref_len": 230,
+                "sacrebleu": 0.4297374729981456,
+                "score": 0.4297374729981456,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.30739045331930365,
+                "score_ci_high": 0.5954313392008956,
+                "sacrebleu_ci_low": 0.30739045331930365,
+                "sacrebleu_ci_high": 0.5954313392008956
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    99,
+                    65,
+                    44
+                ],
+                "totals": [
+                    238,
+                    232,
+                    226,
+                    220
+                ],
+                "precisions": [
+                    0.6932773109243697,
+                    0.4267241379310345,
+                    0.28761061946902655,
+                    0.2
+                ],
+                "bp": 0.9792107358732394,
+                "sys_len": 238,
+                "ref_len": 243,
+                "sacrebleu": 0.35367018032587716,
+                "score": 0.35367018032587716,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3016964479711889,
+                "score_ci_high": 0.4034179481814929,
+                "sacrebleu_ci_low": 0.3016964479711889,
+                "sacrebleu_ci_high": 0.4034179481814929
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    168,
+                    129,
+                    99,
+                    75
+                ],
+                "totals": [
+                    215,
+                    209,
+                    203,
+                    197
+                ],
+                "precisions": [
+                    0.7813953488372093,
+                    0.6172248803827751,
+                    0.4876847290640394,
+                    0.3807106598984772
+                ],
+                "bp": 1.0,
+                "sys_len": 215,
+                "ref_len": 208,
+                "sacrebleu": 0.5470312162394166,
+                "score": 0.5470312162394166,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4764122852102197,
+                "score_ci_high": 0.6508738326325866,
+                "sacrebleu_ci_low": 0.4764122852102197,
+                "sacrebleu_ci_high": 0.6508738326325866
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    143,
+                    86,
+                    60,
+                    42
+                ],
+                "totals": [
+                    215,
+                    209,
+                    203,
+                    197
+                ],
+                "precisions": [
+                    0.6651162790697674,
+                    0.41148325358851673,
+                    0.2955665024630542,
+                    0.2131979695431472
+                ],
+                "bp": 1.0,
+                "sys_len": 215,
+                "ref_len": 208,
+                "sacrebleu": 0.36238649527066064,
+                "score": 0.36238649527066064,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20955142870882296,
+                "score_ci_high": 0.5831549950186898,
+                "sacrebleu_ci_low": 0.20955142870882296,
+                "sacrebleu_ci_high": 0.5831549950186898
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    131,
+                    74,
+                    45,
+                    31
+                ],
+                "totals": [
+                    194,
+                    188,
+                    182,
+                    176
+                ],
+                "precisions": [
+                    0.6752577319587628,
+                    0.39361702127659576,
+                    0.24725274725274726,
+                    0.17613636363636365
+                ],
+                "bp": 0.9303774188371497,
+                "sys_len": 194,
+                "ref_len": 208,
+                "sacrebleu": 0.30517050622006836,
+                "score": 0.30517050622006836,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1936778777853124,
+                "score_ci_high": 0.4521715400303785,
+                "sacrebleu_ci_low": 0.1936778777853124,
+                "sacrebleu_ci_high": 0.4521715400303785
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    167,
+                    128,
+                    100,
+                    80
+                ],
+                "totals": [
+                    211,
+                    205,
+                    199,
+                    193
+                ],
+                "precisions": [
+                    0.7914691943127963,
+                    0.624390243902439,
+                    0.5025125628140704,
+                    0.41450777202072536
+                ],
+                "bp": 1.0,
+                "sys_len": 211,
+                "ref_len": 208,
+                "sacrebleu": 0.5664250237033246,
+                "score": 0.5664250237033246,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.4703853821459762,
+                "score_ci_high": 0.6458520638777493,
+                "sacrebleu_ci_low": 0.4703853821459762,
+                "sacrebleu_ci_high": 0.6458520638777493
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    160,
+                    112,
+                    79,
+                    58
+                ],
+                "totals": [
+                    226,
+                    220,
+                    214,
+                    208
+                ],
+                "precisions": [
+                    0.7079646017699115,
+                    0.509090909090909,
+                    0.36915887850467294,
+                    0.27884615384615385
+                ],
+                "bp": 1.0,
+                "sys_len": 226,
+                "ref_len": 208,
+                "sacrebleu": 0.4388804297038792,
+                "score": 0.4388804297038792,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3074637309571057,
+                "score_ci_high": 0.5696800272393862,
+                "sacrebleu_ci_low": 0.3074637309571057,
+                "sacrebleu_ci_high": 0.5696800272393862
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    151,
+                    97,
+                    62,
+                    42
+                ],
+                "totals": [
+                    216,
+                    210,
+                    204,
+                    198
+                ],
+                "precisions": [
+                    0.6990740740740741,
+                    0.4619047619047619,
+                    0.30392156862745096,
+                    0.2121212121212121
+                ],
+                "bp": 1.0,
+                "sys_len": 216,
+                "ref_len": 208,
+                "sacrebleu": 0.37984403828565183,
+                "score": 0.37984403828565183,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2939549007299014,
+                "score_ci_high": 0.539072297051574,
+                "sacrebleu_ci_low": 0.2939549007299014,
+                "sacrebleu_ci_high": 0.539072297051574
+            },
+            "score": 0.42425093288480964,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.6368761936671354,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/{2025-07-02T17-33-41_evaluation_results.json → 2025-07-03T11-22-55_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T21:33:37.582340Z",
+        "timestamp_utc": "2025-07-03T15:22:51.072452Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -292,57 +292,57 @@
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.5117647058823529,
-                "score": 0.5117647058823529,
+                "llama_3_70b_instruct_template_arena_hard": 0.43209876543209874,
+                "score": 0.43209876543209874,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5117647058823529,
+            "score": 0.43209876543209874,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.46153846153846156,
-                "f1_Organization": 0.15,
-                "f1_Location": 0.16216216216216217,
-                "f1_macro": 0.2579002079002079,
-                "recall_macro": 0.20781573498964803,
-                "precision_macro": 0.3477564102564103,
-                "in_classes_support": 0.8367346938775511,
-                "f1_micro": 0.24193548387096772,
-                "recall_micro": 0.2,
-                "precision_micro": 0.30612244897959184,
-                "score": 0.24193548387096772,
+                "f1_Person": 0.8695652173913043,
+                "f1_Organization": 0.7384615384615384,
+                "f1_Location": 0.8085106382978724,
+                "f1_macro": 0.8055124647169051,
+                "recall_macro": 0.8394582470669426,
+                "precision_macro": 0.781433607520564,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.7974683544303797,
+                "recall_micro": 0.84,
+                "precision_micro": 0.7590361445783133,
+                "score": 0.7974683544303797,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.15613134886988791,
-                "score_ci_high": 0.34768110519026457,
-                "f1_micro_ci_low": 0.15613134886988791,
-                "f1_micro_ci_high": 0.34768110519026457
+                "score_ci_low": 0.7370661700501011,
+                "score_ci_high": 0.8625005221303097,
+                "f1_micro_ci_low": 0.7370661700501011,
+                "f1_micro_ci_high": 0.8625005221303097
             },
-            "score": 0.24193548387096772,
+            "score": 0.7974683544303797,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 0.7142857142857143,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.42857142857142855,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
@@ -376,13 +376,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.5714285714285714,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
@@ -406,13 +406,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 0.7142857142857143,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.42857142857142855,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
@@ -465,90 +465,90 @@
                 "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
-            "score": 0.6836734693877551,
+            "score": 0.6632653061224489,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.8168831168831169,
-                "f1_suggestive": 0.7272727272727273,
+                "f1_macro": 0.7448051948051948,
+                "f1_suggestive": 0.5454545454545454,
                 "f1_generic": 1.0,
                 "f1_fanciful": 0.8571428571428571,
-                "f1_descriptive": 0.75,
+                "f1_descriptive": 0.5714285714285714,
                 "f1_arbitrary": 0.75,
-                "f1_macro_ci_low": 0.6234634793664356,
-                "f1_macro_ci_high": 0.9631551676258282,
+                "f1_macro_ci_low": 0.5498662790687274,
+                "f1_macro_ci_high": 0.9225357772974475,
+                "score_name": "f1_micro",
+                "score": 0.717948717948718,
+                "score_ci_high": 0.9,
+                "score_ci_low": 0.47368421052631576,
+                "num_of_instances": 20,
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.717948717948718,
+                "f1_micro_ci_low": 0.47368421052631576,
+                "f1_micro_ci_high": 0.9
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.7333333333333334,
+                "f1_no": 0.8666666666666667,
+                "f1_yes": 0.6,
+                "f1_macro_ci_low": 0.4666666666666667,
+                "f1_macro_ci_high": 0.96223632692803,
                 "score_name": "f1_micro",
                 "score": 0.8,
                 "score_ci_high": 0.95,
-                "score_ci_low": 0.5594941844349111,
+                "score_ci_low": 0.5971324299664202,
                 "num_of_instances": 20,
                 "accuracy": 0.8,
-                "accuracy_ci_low": 0.5730020405512491,
+                "accuracy_ci_low": 0.5971324299664202,
                 "accuracy_ci_high": 0.95,
                 "f1_micro": 0.8,
-                "f1_micro_ci_low": 0.5594941844349111,
+                "f1_micro_ci_low": 0.5971324299664202,
                 "f1_micro_ci_high": 0.95
             },
-            "legalbench_corporate_lobbying": {
-                "f1_macro": 0.6000000000000001,
-                "f1_no": 0.8,
-                "f1_yes": 0.4,
-                "f1_macro_ci_low": 0.375,
-                "f1_macro_ci_high": 0.8857142857142857,
-                "score_name": "f1_micro",
-                "score": 0.7,
-                "score_ci_high": 0.85,
-                "score_ci_low": 0.45,
-                "num_of_instances": 20,
-                "accuracy": 0.7,
-                "accuracy_ci_low": 0.45,
-                "accuracy_ci_high": 0.85,
-                "f1_micro": 0.7,
-                "f1_micro_ci_low": 0.45,
-                "f1_micro_ci_high": 0.85
-            },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.20578231292517007,
-                "f1_conclusion": 0.25,
+                "f1_macro": 0.2006802721088435,
+                "f1_conclusion": 0.2857142857142857,
+                "f1_analysis": 0.5,
                 "f1_decree": 0.0,
-                "f1_rule": 0.0,
                 "f1_issue": 0.2857142857142857,
-                "f1_analysis": 0.3333333333333333,
-                "f1_facts": 0.5714285714285714,
+                "f1_facts": 0.3333333333333333,
                 "f1_procedural history": 0.0,
-                "f1_macro_ci_low": 0.07142857142857142,
-                "f1_macro_ci_high": 0.3891151448663562,
+                "f1_rule": 0.0,
+                "f1_macro_ci_low": 0.07434693648030322,
+                "f1_macro_ci_high": 0.41439679265221946,
                 "score_name": "f1_micro",
-                "score": 0.2702702702702703,
-                "score_ci_high": 0.47368421052631576,
-                "score_ci_low": 0.0940372444530111,
+                "score": 0.2777777777777778,
+                "score_ci_high": 0.5,
+                "score_ci_low": 0.10739242693044677,
                 "num_of_instances": 20,
                 "accuracy": 0.25,
                 "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.48260896645204526,
-                "f1_micro": 0.2702702702702703,
-                "f1_micro_ci_low": 0.0940372444530111,
-                "f1_micro_ci_high": 0.47368421052631576
+                "accuracy_ci_high": 0.5,
+                "f1_micro": 0.2777777777777778,
+                "f1_micro_ci_low": 0.10739242693044677,
+                "f1_micro_ci_high": 0.5
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.48717948717948717,
-                "f1_yes": 0.6666666666666666,
-                "f1_no": 0.3076923076923077,
-                "f1_macro_ci_low": 0.3103448275862069,
-                "f1_macro_ci_high": 0.7619047619047619,
+                "f1_macro": 0.45054945054945056,
+                "f1_yes": 0.6153846153846154,
+                "f1_no": 0.2857142857142857,
+                "f1_macro_ci_low": 0.27083333333333337,
+                "f1_macro_ci_high": 0.7150997150997151,
                 "score_name": "f1_micro",
-                "score": 0.55,
-                "score_ci_high": 0.75,
-                "score_ci_low": 0.3,
+                "score": 0.5,
+                "score_ci_high": 0.7,
+                "score_ci_low": 0.28933563893775816,
                 "num_of_instances": 20,
-                "accuracy": 0.55,
-                "accuracy_ci_low": 0.3,
-                "accuracy_ci_high": 0.75,
-                "f1_micro": 0.55,
-                "f1_micro_ci_low": 0.3,
-                "f1_micro_ci_high": 0.75
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.28933563893775816,
+                "accuracy_ci_high": 0.7,
+                "f1_micro": 0.5,
+                "f1_micro_ci_low": 0.28933563893775816,
+                "f1_micro_ci_high": 0.7
             },
             "legalbench_proa": {
                 "f1_macro": 0.949874686716792,
@@ -568,84 +568,84 @@
                 "f1_micro_ci_low": 0.7480573644337235,
                 "f1_micro_ci_high": 1.0
             },
-            "score": 0.654054054054054,
+            "score": 0.6491452991452992,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.6262546440487619,
-                "f1_cars": 1.0,
+                "f1_macro": 0.6360263347763347,
+                "f1_cars": 0.9090909090909091,
                 "f1_windows x": 0.3333333333333333,
-                "f1_computer graphics": 0.5882352941176471,
-                "f1_atheism": 0.5,
+                "f1_computer graphics": 0.625,
+                "f1_atheism": 0.5714285714285714,
                 "f1_religion": 0.0,
                 "f1_medicine": 0.8571428571428571,
                 "f1_christianity": 0.8571428571428571,
-                "f1_microsoft windows": 0.6666666666666666,
+                "f1_for sale": 0.5714285714285714,
+                "f1_microsoft windows": 0.9090909090909091,
                 "f1_middle east": 0.5,
-                "f1_motorcycles": 0.7272727272727273,
-                "f1_pc hardware": 0.8,
+                "f1_motorcycles": 0.6,
+                "f1_pc hardware": 0.6666666666666666,
                 "f1_mac hardware": 0.8,
                 "f1_electronics": 0.5,
-                "f1_for sale": 0.5,
                 "f1_guns": 0.6,
-                "f1_space": 0.75,
+                "f1_politics": 0.3333333333333333,
+                "f1_space": 0.8888888888888888,
                 "f1_cryptography": 0.4,
-                "f1_baseball": 0.9230769230769231,
+                "f1_baseball": 0.9090909090909091,
                 "f1_hockey": 0.8888888888888888,
-                "f1_politics": 0.3333333333333333,
-                "f1_macro_ci_low": 0.5374902873696477,
-                "f1_macro_ci_high": 0.7404797748575771,
+                "f1_macro_ci_low": 0.5511216451732288,
+                "f1_macro_ci_high": 0.7430408448382602,
                 "score_name": "f1_micro",
-                "score": 0.6593406593406593,
-                "score_ci_high": 0.7553191489361702,
-                "score_ci_low": 0.5556888171204825,
+                "score": 0.6629834254143646,
+                "score_ci_high": 0.7472339245831151,
+                "score_ci_low": 0.5537799874567927,
                 "num_of_instances": 100,
                 "accuracy": 0.6,
                 "accuracy_ci_low": 0.5,
-                "accuracy_ci_high": 0.7,
-                "f1_micro": 0.6593406593406593,
-                "f1_micro_ci_low": 0.5556888171204825,
-                "f1_micro_ci_high": 0.7553191489361702
+                "accuracy_ci_high": 0.69,
+                "f1_micro": 0.6629834254143646,
+                "f1_micro_ci_low": 0.5537799874567927,
+                "f1_micro_ci_high": 0.7472339245831151
             },
-            "score": 0.6593406593406593,
+            "score": 0.6629834254143646,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.77593809453606,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9264705882352942,
+                "f1_macro": 0.774627043799976,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9172932330827067,
                 "f1_credit card or prepaid card": 0.7368421052631579,
                 "f1_money transfer or virtual currency or money service": 0.8,
                 "f1_mortgage": 0.6666666666666666,
                 "f1_debt collection": 0.7777777777777778,
                 "f1_checking or savings account": 0.8571428571428571,
                 "f1_payday loan or title loan or personal loan": 0.6666666666666666,
-                "f1_macro_ci_low": 0.5600914349154559,
-                "f1_macro_ci_high": 0.879706193608078,
+                "f1_macro_ci_low": 0.575934991917505,
+                "f1_macro_ci_high": 0.8809011779404143,
                 "score_name": "f1_micro",
-                "score": 0.8787878787878788,
-                "score_ci_high": 0.9292929292929293,
-                "score_ci_low": 0.7968657096330831,
+                "score": 0.8717948717948718,
+                "score_ci_high": 0.9238578680203046,
+                "score_ci_low": 0.78640508344599,
                 "num_of_instances": 100,
-                "accuracy": 0.87,
-                "accuracy_ci_low": 0.7994226224456547,
-                "accuracy_ci_high": 0.93,
-                "f1_micro": 0.8787878787878788,
-                "f1_micro_ci_low": 0.7968657096330831,
-                "f1_micro_ci_high": 0.9292929292929293
+                "accuracy": 0.85,
+                "accuracy_ci_low": 0.77,
+                "accuracy_ci_high": 0.91,
+                "f1_micro": 0.8717948717948718,
+                "f1_micro_ci_low": 0.78640508344599,
+                "f1_micro_ci_high": 0.9238578680203046
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.7822360248447204,
-                "f1_mortgages and loans": 0.8333333333333334,
+                "f1_macro": 0.7825559947299078,
+                "f1_mortgages and loans": 0.8695652173913043,
                 "f1_credit card": 0.782608695652174,
                 "f1_debt collection": 0.7,
-                "f1_credit reporting": 0.7619047619047619,
+                "f1_credit reporting": 0.7272727272727273,
                 "f1_retail banking": 0.8333333333333334,
-                "f1_macro_ci_low": 0.6477274108742415,
-                "f1_macro_ci_high": 0.8928244805829546,
+                "f1_macro_ci_low": 0.6554182687947065,
+                "f1_macro_ci_high": 0.8950729595163965,
                 "score_name": "f1_micro",
                 "score": 0.78,
                 "score_ci_high": 0.88,
@@ -658,80 +658,80 @@
                 "f1_micro_ci_low": 0.64,
                 "f1_micro_ci_high": 0.88
             },
-            "score": 0.8293939393939394,
+            "score": 0.8258974358974359,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.26,
-                "score": 0.26,
+                "execution_accuracy": 0.26,
+                "program_accuracy": 0.27,
+                "score": 0.27,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.24,
-                "program_accuracy_ci_low": 0.18,
-                "program_accuracy_ci_high": 0.34578514480330114,
-                "score_ci_low": 0.18,
-                "score_ci_high": 0.34578514480330114,
-                "execution_accuracy_ci_low": 0.16,
-                "execution_accuracy_ci_high": 0.33
+                "execution_accuracy_ci_low": 0.18,
+                "execution_accuracy_ci_high": 0.36,
+                "program_accuracy_ci_low": 0.19,
+                "program_accuracy_ci_high": 0.36,
+                "score_ci_low": 0.19,
+                "score_ci_high": 0.36
             },
-            "score": 0.26,
+            "score": 0.27,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.5129557312390282,
-                "recall": 0.620033623550633,
-                "f1": 0.5210801839331447,
-                "precision_ci_low": 0.47529222906921964,
-                "precision_ci_high": 0.5538659877251515,
-                "recall_ci_low": 0.5796829912906432,
-                "recall_ci_high": 0.6632745686404776,
-                "f1_ci_low": 0.48926316535354414,
-                "f1_ci_high": 0.5559446648824712,
+                "precision": 0.5331803217904779,
+                "recall": 0.6162591761810932,
+                "f1": 0.5317474758084658,
+                "precision_ci_low": 0.49517557669164985,
+                "precision_ci_high": 0.5720982419484572,
+                "recall_ci_low": 0.5754133026164426,
+                "recall_ci_high": 0.6573336562511775,
+                "f1_ci_low": 0.501458251561677,
+                "f1_ci_high": 0.5625425236205234,
                 "score_name": "f1",
-                "score": 0.5210801839331447,
-                "score_ci_high": 0.5559446648824712,
-                "score_ci_low": 0.48926316535354414,
+                "score": 0.5317474758084658,
+                "score_ci_high": 0.5625425236205234,
+                "score_ci_low": 0.501458251561677,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.7023521369695663,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7295124924182892,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6883632922172547,
-                "faithfullness_f1_token_overlap": 0.4012430193036613,
-                "faithfullness_recall_token_overlap": 0.29495342603130686,
-                "faithfullness_precision_token_overlap": 0.7420458881815755,
-                "correctness_f1_token_overlap": 0.5210801839331447,
-                "correctness_recall_token_overlap": 0.620033623550633,
-                "correctness_precision_token_overlap": 0.5129557312390282
+                "correctness_f1_bert_score.deberta_large_mnli": 0.703714978992939,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7273174220323563,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6931151136755943,
+                "faithfullness_f1_token_overlap": 0.38943632812862744,
+                "faithfullness_recall_token_overlap": 0.2823300548933722,
+                "faithfullness_precision_token_overlap": 0.7431638772011331,
+                "correctness_f1_token_overlap": 0.5317474758084658,
+                "correctness_recall_token_overlap": 0.6162591761810932,
+                "correctness_precision_token_overlap": 0.5331803217904779
             },
-            "score": 0.5210801839331447,
+            "score": 0.5317474758084658,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.58,
-                "accuracy_ci_low": 0.47,
-                "accuracy_ci_high": 0.66,
+                "accuracy": 0.57,
+                "accuracy_ci_low": 0.46198165422712656,
+                "accuracy_ci_high": 0.65,
                 "score_name": "accuracy",
-                "score": 0.58,
-                "score_ci_high": 0.66,
-                "score_ci_low": 0.47,
+                "score": 0.57,
+                "score_ci_high": 0.65,
+                "score_ci_low": 0.46198165422712656,
                 "num_of_instances": 100
             },
             "openbook_qa": {
                 "accuracy": 0.93,
-                "accuracy_ci_low": 0.86,
+                "accuracy_ci_low": 0.87,
                 "accuracy_ci_high": 0.97,
                 "score_name": "accuracy",
                 "score": 0.93,
                 "score_ci_high": 0.97,
-                "score_ci_low": 0.86,
+                "score_ci_low": 0.87,
                 "num_of_instances": 100
             },
-            "score": 0.755,
+            "score": 0.75,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -741,66 +741,66 @@
                 "severity_high": 0.0,
                 "severity_medium": 0.0,
                 "severity_low": 100.0,
-                "category_harmful_info": 0.8359064126002885,
-                "category_deception": 0.8486584005030718,
-                "category_explicit_content": 0.8562041158326948,
-                "category_violence": 0.8001905072184782,
-                "category_discrimination": 0.760403573513031,
-                "category_substance_abuse": 0.8383491781022814,
-                "category_pii": 0.7943065918430134,
-                "safety": 0.820630256657799,
-                "safety_ci_low": 0.8016362653713062,
-                "safety_ci_high": 0.8386065730632751,
+                "category_harmful_info": 0.8221827392569846,
+                "category_deception": 0.8071670434390654,
+                "category_explicit_content": 0.8828472378428533,
+                "category_violence": 0.7895851463789031,
+                "category_discrimination": 0.7562157607504298,
+                "category_substance_abuse": 0.8583730585045284,
+                "category_pii": 0.8396110937551216,
+                "safety": 0.8135096281559931,
+                "safety_ci_low": 0.7967702393625679,
+                "safety_ci_high": 0.8310506142233636,
                 "score_name": "safety",
-                "score": 0.820630256657799,
-                "score_ci_high": 0.8386065730632751,
-                "score_ci_low": 0.8016362653713062,
+                "score": 0.8135096281559931,
+                "score_ci_high": 0.8310506142233636,
+                "score_ci_low": 0.7967702393625679,
                 "num_of_instances": 100
             },
-            "score": 0.820630256657799,
+            "score": 0.8135096281559931,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.42292208897681,
-                "rougeL": 0.29354791609951486,
-                "score": 0.29354791609951486,
+                "rouge1": 0.4261297102014613,
+                "rougeLsum": 0.36938097206123816,
+                "rouge2": 0.2036872065921589,
+                "rougeL": 0.2968550225185778,
+                "score": 0.2968550225185778,
                 "score_name": "rougeL",
-                "rouge2": 0.20392519030368167,
-                "rougeLsum": 0.3673127942632855,
-                "rouge1_ci_low": 0.39963192681082876,
-                "rouge1_ci_high": 0.4458425564295312,
-                "rougeL_ci_low": 0.27738686729862344,
-                "rougeL_ci_high": 0.31213298595537586,
-                "score_ci_low": 0.27738686729862344,
-                "score_ci_high": 0.31213298595537586,
-                "rouge2_ci_low": 0.18675933398879171,
-                "rouge2_ci_high": 0.21963362631469394,
-                "rougeLsum_ci_low": 0.3458152710057813,
-                "rougeLsum_ci_high": 0.3886449359967606
+                "rouge1_ci_low": 0.40335430952139306,
+                "rouge1_ci_high": 0.44823436472459277,
+                "rougeLsum_ci_low": 0.3493633172912225,
+                "rougeLsum_ci_high": 0.3905259807810269,
+                "rouge2_ci_low": 0.1891962367215942,
+                "rouge2_ci_high": 0.22028254768837505,
+                "rougeL_ci_low": 0.27992380389675553,
+                "rougeL_ci_high": 0.3154054172474221,
+                "score_ci_low": 0.27992380389675553,
+                "score_ci_high": 0.3154054172474221
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.110679309226253,
-                "rougeL": 0.08281233434176384,
-                "score": 0.08281233434176384,
+                "rouge1": 0.11562725478537023,
+                "rougeLsum": 0.09465508243515998,
+                "rouge2": 0.017708600304049516,
+                "rougeL": 0.08457970229678397,
+                "score": 0.08457970229678397,
                 "score_name": "rougeL",
-                "rouge2": 0.01645093034827128,
-                "rougeLsum": 0.09191967073004667,
-                "rouge1_ci_low": 0.09478192397881031,
-                "rouge1_ci_high": 0.12658475093743515,
-                "rougeL_ci_low": 0.07146606223647664,
-                "rougeL_ci_high": 0.09451527487759007,
-                "score_ci_low": 0.07146606223647664,
-                "score_ci_high": 0.09451527487759007,
-                "rouge2_ci_low": 0.011265397606465145,
-                "rouge2_ci_high": 0.022819262497260188,
-                "rougeLsum_ci_low": 0.07977530131627288,
-                "rougeLsum_ci_high": 0.10477307872763918
+                "rouge1_ci_low": 0.09872400500532583,
+                "rouge1_ci_high": 0.13225162304183855,
+                "rougeLsum_ci_low": 0.08102101648826818,
+                "rougeLsum_ci_high": 0.1079514615631145,
+                "rouge2_ci_low": 0.01291561124547185,
+                "rouge2_ci_high": 0.02325190774651309,
+                "rougeL_ci_low": 0.07285237512783484,
+                "rougeL_ci_high": 0.09622345560028686,
+                "score_ci_low": 0.07285237512783484,
+                "score_ci_high": 0.09622345560028686
             },
-            "score": 0.18818012522063934,
+            "score": 0.1907173624076809,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    160,
-                    125,
-                    102,
-                    86
+                    162,
+                    117,
+                    85,
+                    63
                 ],
                 "totals": [
-                    207,
-                    201,
-                    195,
-                    189
+                    221,
+                    215,
+                    209,
+                    203
                 ],
                 "precisions": [
-                    0.7729468599033816,
-                    0.6218905472636816,
-                    0.5230769230769231,
-                    0.455026455026455
+                    0.7330316742081447,
+                    0.5441860465116278,
+                    0.4066985645933014,
+                    0.3103448275862069
                 ],
-                "bp": 0.9951807322415573,
-                "sys_len": 207,
+                "bp": 1.0,
+                "sys_len": 221,
                 "ref_len": 208,
-                "sacrebleu": 0.5787865217176954,
-                "score": 0.5787865217176954,
+                "sacrebleu": 0.4736928434500847,
+                "score": 0.4736928434500847,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.29885182944464195,
-                "score_ci_high": 0.7257450793573996,
-                "sacrebleu_ci_low": 0.29885182944464195,
-                "sacrebleu_ci_high": 0.7257450793573996
+                "score_ci_low": 0.2928408927128658,
+                "score_ci_high": 0.551671370510479,
+                "sacrebleu_ci_low": 0.2928408927128658,
+                "sacrebleu_ci_high": 0.551671370510479
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    150,
-                    99,
-                    67,
-                    49
+                    142,
+                    89,
+                    58,
+                    39
                 ],
                 "totals": [
-                    216,
-                    210,
                     204,
-                    198
+                    198,
+                    192,
+                    186
                 ],
                 "precisions": [
-                    0.6944444444444444,
-                    0.4714285714285715,
-                    0.3284313725490196,
-                    0.2474747474747475
+                    0.696078431372549,
+                    0.4494949494949495,
+                    0.3020833333333333,
+                    0.20967741935483872
                 ],
-                "bp": 1.0,
-                "sys_len": 216,
+                "bp": 0.9805831403241088,
+                "sys_len": 204,
                 "ref_len": 208,
-                "sacrebleu": 0.40388454349139896,
-                "score": 0.40388454349139896,
+                "sacrebleu": 0.3679169337842913,
+                "score": 0.3679169337842913,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3120778874943009,
-                "score_ci_high": 0.5764421068033629,
-                "sacrebleu_ci_low": 0.3120778874943009,
-                "sacrebleu_ci_high": 0.5764421068033629
+                "score_ci_low": 0.2502353509368021,
+                "score_ci_high": 0.5243739849514055,
+                "sacrebleu_ci_low": 0.2502353509368021,
+                "sacrebleu_ci_high": 0.5243739849514055
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    120,
-                    70,
-                    42,
-                    23
+                    111,
+                    58,
+                    33,
+                    16
                 ],
                 "totals": [
-                    201,
                     195,
                     189,
-                    183
+                    183,
+                    177
                 ],
                 "precisions": [
-                    0.5970149253731344,
-                    0.358974358974359,
-                    0.2222222222222222,
-                    0.12568306010928962
+                    0.5692307692307692,
+                    0.30687830687830686,
+                    0.18032786885245902,
+                    0.0903954802259887
                 ],
-                "bp": 0.960980660057086,
-                "sys_len": 201,
+                "bp": 0.9307217935222629,
+                "sys_len": 195,
                 "ref_len": 209,
-                "sacrebleu": 0.2672962463170595,
-                "score": 0.2672962463170595,
+                "sacrebleu": 0.21499873971480102,
+                "score": 0.21499873971480102,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.20195568594939164,
-                "score_ci_high": 0.3449601603470983,
-                "sacrebleu_ci_low": 0.20195568594939164,
-                "sacrebleu_ci_high": 0.3449601603470983
+                "score_ci_low": 0.10854451987808533,
+                "score_ci_high": 0.3074151801090054,
+                "sacrebleu_ci_low": 0.10854451987808533,
+                "sacrebleu_ci_high": 0.3074151801090054
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    146,
-                    89,
-                    56,
-                    39
+                    143,
+                    88,
+                    58,
+                    42
                 ],
                 "totals": [
-                    227,
-                    221,
-                    215,
-                    209
+                    220,
+                    214,
+                    208,
+                    202
                 ],
                 "precisions": [
-                    0.6431718061674009,
-                    0.40271493212669685,
-                    0.26046511627906976,
-                    0.18660287081339713
+                    0.65,
+                    0.4112149532710281,
+                    0.27884615384615385,
+                    0.20792079207920794
                 ],
                 "bp": 1.0,
-                "sys_len": 227,
+                "sys_len": 220,
                 "ref_len": 216,
-                "sacrebleu": 0.3349640160520034,
-                "score": 0.3349640160520034,
+                "sacrebleu": 0.352826509901891,
+                "score": 0.352826509901891,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23842068549265574,
-                "score_ci_high": 0.4242989412556468,
-                "sacrebleu_ci_low": 0.23842068549265574,
-                "sacrebleu_ci_high": 0.4242989412556468
+                "score_ci_low": 0.2425322687880856,
+                "score_ci_high": 0.4774115974551837,
+                "sacrebleu_ci_low": 0.2425322687880856,
+                "sacrebleu_ci_high": 0.4774115974551837
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    188,
-                    145,
-                    115,
-                    93
+                    187,
+                    140,
+                    106,
+                    85
                 ],
                 "totals": [
-                    237,
-                    231,
-                    225,
-                    219
+                    241,
+                    235,
+                    229,
+                    223
                 ],
                 "precisions": [
-                    0.7932489451476793,
-                    0.6277056277056277,
-                    0.5111111111111112,
-                    0.4246575342465754
+                    0.7759336099585061,
+                    0.5957446808510638,
+                    0.46288209606986896,
+                    0.3811659192825112
                 ],
                 "bp": 1.0,
-                "sys_len": 237,
+                "sys_len": 241,
                 "ref_len": 235,
-                "sacrebleu": 0.5733633387244307,
-                "score": 0.5733633387244307,
+                "sacrebleu": 0.5344010413403237,
+                "score": 0.5344010413403237,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.5002460947078978,
-                "score_ci_high": 0.6377787499465616,
-                "sacrebleu_ci_low": 0.5002460947078978,
-                "sacrebleu_ci_high": 0.6377787499465616
+                "score_ci_low": 0.4475998249972296,
+                "score_ci_high": 0.6492252552812806,
+                "sacrebleu_ci_low": 0.4475998249972296,
+                "sacrebleu_ci_high": 0.6492252552812806
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    168,
-                    95,
-                    56,
-                    36
+                    164,
+                    93,
+                    57,
+                    38
                 ],
                 "totals": [
-                    335,
-                    329,
-                    323,
-                    317
+                    278,
+                    272,
+                    266,
+                    260
                 ],
                 "precisions": [
-                    0.5014925373134329,
-                    0.2887537993920973,
-                    0.17337461300309598,
-                    0.11356466876971609
+                    0.5899280575539568,
+                    0.3419117647058823,
+                    0.21428571428571427,
+                    0.14615384615384616
                 ],
                 "bp": 1.0,
-                "sys_len": 335,
+                "sys_len": 278,
                 "ref_len": 249,
-                "sacrebleu": 0.23107620759127706,
-                "score": 0.23107620759127706,
+                "sacrebleu": 0.2819221125537144,
+                "score": 0.2819221125537144,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1680722832692475,
-                "score_ci_high": 0.3140076000318745,
-                "sacrebleu_ci_low": 0.1680722832692475,
-                "sacrebleu_ci_high": 0.3140076000318745
+                "score_ci_low": 0.2083152805325296,
+                "score_ci_high": 0.361735467626507,
+                "sacrebleu_ci_low": 0.2083152805325296,
+                "sacrebleu_ci_high": 0.361735467626507
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    182,
-                    140,
-                    115,
-                    96
+                    177,
+                    126,
+                    104,
+                    88
                 ],
                 "totals": [
-                    227,
-                    221,
-                    215,
-                    209
+                    226,
+                    220,
+                    214,
+                    208
                 ],
                 "precisions": [
-                    0.801762114537445,
-                    0.6334841628959276,
-                    0.5348837209302326,
-                    0.4593301435406698
+                    0.7831858407079646,
+                    0.5727272727272728,
+                    0.48598130841121495,
+                    0.4230769230769231
                 ],
                 "bp": 1.0,
-                "sys_len": 227,
+                "sys_len": 226,
                 "ref_len": 222,
-                "sacrebleu": 0.5943488203126417,
-                "score": 0.5943488203126417,
+                "sacrebleu": 0.5510777780526783,
+                "score": 0.5510777780526783,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.5234160972918542,
-                "score_ci_high": 0.7151947718325412,
-                "sacrebleu_ci_low": 0.5234160972918542,
-                "sacrebleu_ci_high": 0.7151947718325412
+                "score_ci_low": 0.4632937438879833,
+                "score_ci_high": 0.6949332476008023,
+                "sacrebleu_ci_low": 0.4632937438879833,
+                "sacrebleu_ci_high": 0.6949332476008023
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    163,
-                    122,
-                    93,
-                    73
+                    159,
+                    117,
+                    88,
+                    70
                 ],
                 "totals": [
-                    229,
-                    223,
-                    217,
-                    211
+                    231,
+                    225,
+                    219,
+                    213
                 ],
                 "precisions": [
-                    0.7117903930131004,
-                    0.547085201793722,
-                    0.42857142857142855,
-                    0.3459715639810427
+                    0.6883116883116882,
+                    0.52,
+                    0.4018264840182648,
+                    0.3286384976525822
                 ],
-                "bp": 0.9956427084340843,
-                "sys_len": 229,
+                "bp": 1.0,
+                "sys_len": 231,
                 "ref_len": 230,
-                "sacrebleu": 0.4880577291073812,
-                "score": 0.4880577291073812,
+                "sacrebleu": 0.46626881559414857,
+                "score": 0.46626881559414857,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3992120050572284,
-                "score_ci_high": 0.5996458265914114,
-                "sacrebleu_ci_low": 0.3992120050572284,
-                "sacrebleu_ci_high": 0.5996458265914114
+                "score_ci_low": 0.38426649851701006,
+                "score_ci_high": 0.5675525143212888,
+                "sacrebleu_ci_low": 0.38426649851701006,
+                "sacrebleu_ci_high": 0.5675525143212888
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    165,
-                    93,
-                    55,
-                    35
+                    156,
+                    91,
+                    57,
+                    36
                 ],
                 "totals": [
+                    234,
                     228,
                     222,
-                    216,
-                    210
+                    216
                 ],
                 "precisions": [
-                    0.7236842105263157,
-                    0.41891891891891897,
-                    0.2546296296296296,
+                    0.6666666666666667,
+                    0.3991228070175438,
+                    0.2567567567567568,
                     0.16666666666666669
                 ],
-                "bp": 0.936327965220313,
-                "sys_len": 228,
+                "bp": 0.9622687143632572,
+                "sys_len": 234,
                 "ref_len": 243,
-                "sacrebleu": 0.3153458967950177,
-                "score": 0.3153458967950177,
+                "sacrebleu": 0.31433507572000613,
+                "score": 0.31433507572000613,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2606471545650505,
-                "score_ci_high": 0.3987290270936019,
-                "sacrebleu_ci_low": 0.2606471545650505,
-                "sacrebleu_ci_high": 0.3987290270936019
+                "score_ci_low": 0.24701452221728332,
+                "score_ci_high": 0.39765832727161077,
+                "sacrebleu_ci_low": 0.24701452221728332,
+                "sacrebleu_ci_high": 0.39765832727161077
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    166,
-                    126,
-                    92,
-                    66
+                    176,
+                    144,
+                    115,
+                    91
                 ],
                 "totals": [
-                    214,
-                    208,
-                    202,
-                    196
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.7757009345794392,
-                    0.6057692307692308,
-                    0.4554455445544554,
-                    0.33673469387755106
+                    0.8186046511627907,
+                    0.6889952153110047,
+                    0.5665024630541872,
+                    0.4619289340101523
                 ],
                 "bp": 1.0,
-                "sys_len": 214,
+                "sys_len": 215,
                 "ref_len": 208,
-                "sacrebleu": 0.5181213181389714,
-                "score": 0.5181213181389714,
+                "sacrebleu": 0.6198217981505746,
+                "score": 0.6198217981505746,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.42303533208436167,
-                "score_ci_high": 0.5858625425016247,
-                "sacrebleu_ci_low": 0.42303533208436167,
-                "sacrebleu_ci_high": 0.5858625425016247
+                "score_ci_low": 0.511273668285519,
+                "score_ci_high": 0.7301855114871573,
+                "sacrebleu_ci_low": 0.511273668285519,
+                "sacrebleu_ci_high": 0.7301855114871573
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    150,
-                    94,
-                    66,
-                    50
+                    142,
+                    85,
+                    59,
+                    44
                 ],
                 "totals": [
-                    218,
-                    212,
-                    206,
-                    200
+                    204,
+                    198,
+                    192,
+                    186
                 ],
                 "precisions": [
-                    0.6880733944954129,
-                    0.44339622641509435,
-                    0.3203883495145631,
-                    0.25
+                    0.696078431372549,
+                    0.4292929292929293,
+                    0.3072916666666667,
+                    0.23655913978494625
                 ],
-                "bp": 1.0,
-                "sys_len": 218,
+                "bp": 0.9805831403241088,
+                "sys_len": 204,
                 "ref_len": 208,
-                "sacrebleu": 0.3953765163102563,
-                "score": 0.3953765163102563,
+                "sacrebleu": 0.376452364097502,
+                "score": 0.376452364097502,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1804482419851951,
-                "score_ci_high": 0.5539172422857372,
-                "sacrebleu_ci_low": 0.1804482419851951,
-                "sacrebleu_ci_high": 0.5539172422857372
+                "score_ci_low": 0.11704022800592581,
+                "score_ci_high": 0.6173701422128451,
+                "sacrebleu_ci_low": 0.11704022800592581,
+                "sacrebleu_ci_high": 0.6173701422128451
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    135,
-                    77,
-                    46,
-                    34
+                    143,
+                    93,
+                    60,
+                    43
                 ],
                 "totals": [
-                    205,
-                    199,
-                    193,
-                    187
+                    212,
+                    206,
+                    200,
+                    194
                 ],
                 "precisions": [
-                    0.6585365853658537,
-                    0.3869346733668342,
-                    0.23834196891191708,
-                    0.18181818181818182
+                    0.6745283018867925,
+                    0.4514563106796116,
+                    0.3,
+                    0.22164948453608246
                 ],
-                "bp": 0.9854724123463497,
-                "sys_len": 205,
+                "bp": 1.0,
+                "sys_len": 212,
                 "ref_len": 208,
-                "sacrebleu": 0.31945399108502914,
-                "score": 0.31945399108502914,
+                "sacrebleu": 0.3772254378250882,
+                "score": 0.3772254378250882,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18378614895900558,
-                "score_ci_high": 0.5319944061578967,
-                "sacrebleu_ci_low": 0.18378614895900558,
-                "sacrebleu_ci_high": 0.5319944061578967
+                "score_ci_low": 0.2728133483181,
+                "score_ci_high": 0.5562821154882652,
+                "sacrebleu_ci_low": 0.2728133483181,
+                "sacrebleu_ci_high": 0.5562821154882652
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    180,
-                    142,
-                    110,
-                    86
+                    170,
+                    130,
+                    100,
+                    75
                 ],
                 "totals": [
-                    216,
-                    210,
-                    204,
-                    198
+                    220,
+                    214,
+                    208,
+                    202
                 ],
                 "precisions": [
-                    0.8333333333333333,
-                    0.6761904761904762,
-                    0.5392156862745098,
-                    0.4343434343434343
+                    0.7727272727272727,
+                    0.6074766355140188,
+                    0.4807692307692308,
+                    0.3712871287128713
                 ],
                 "bp": 1.0,
-                "sys_len": 216,
+                "sys_len": 220,
                 "ref_len": 208,
-                "sacrebleu": 0.6027273453558246,
-                "score": 0.6027273453558246,
+                "sacrebleu": 0.5380226938344174,
+                "score": 0.5380226938344174,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.48486657280711865,
-                "score_ci_high": 0.6459728814625068,
-                "sacrebleu_ci_low": 0.48486657280711865,
-                "sacrebleu_ci_high": 0.6459728814625068
+                "score_ci_low": 0.39009706347814604,
+                "score_ci_high": 0.6760418516266744,
+                "sacrebleu_ci_low": 0.39009706347814604,
+                "sacrebleu_ci_high": 0.6760418516266744
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
                     159,
-                    107,
-                    77,
-                    58
+                    111,
+                    80,
+                    60
                 ],
                 "totals": [
-                    218,
-                    212,
-                    206,
-                    200
+                    213,
+                    207,
+                    201,
+                    195
                 ],
                 "precisions": [
-                    0.7293577981651376,
-                    0.5047169811320754,
-                    0.3737864077669903,
-                    0.29
+                    0.7464788732394366,
+                    0.5362318840579711,
+                    0.3980099502487562,
+                    0.3076923076923077
                 ],
                 "bp": 1.0,
-                "sys_len": 218,
+                "sys_len": 213,
                 "ref_len": 208,
-                "sacrebleu": 0.44694338363320946,
-                "score": 0.44694338363320946,
+                "sacrebleu": 0.4705385184307256,
+                "score": 0.4705385184307256,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3413111085443379,
-                "score_ci_high": 0.6033707864456245,
-                "sacrebleu_ci_low": 0.3413111085443379,
-                "sacrebleu_ci_high": 0.6033707864456245
+                "score_ci_low": 0.3704730772692346,
+                "score_ci_high": 0.5955255847131556,
+                "sacrebleu_ci_low": 0.3704730772692346,
+                "sacrebleu_ci_high": 0.5955255847131556
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
                     145,
-                    92,
+                    95,
                     60,
-                    42
+                    40
                 ],
                 "totals": [
-                    206,
-                    200,
-                    194,
-                    188
+                    205,
+                    199,
+                    193,
+                    187
                 ],
                 "precisions": [
-                    0.703883495145631,
-                    0.46,
-                    0.30927835051546393,
-                    0.22340425531914893
+                    0.7073170731707318,
+                    0.4773869346733668,
+                    0.31088082901554404,
+                    0.21390374331550802
                 ],
-                "bp": 0.9903382397772544,
-                "sys_len": 206,
+                "bp": 0.9854724123463497,
+                "sys_len": 205,
                 "ref_len": 208,
-                "sacrebleu": 0.383008526679698,
-                "score": 0.383008526679698,
+                "sacrebleu": 0.3814773174854295,
+                "score": 0.3814773174854295,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3224088366860718,
-                "score_ci_high": 0.49162919882110173,
-                "sacrebleu_ci_low": 0.3224088366860718,
-                "sacrebleu_ci_high": 0.49162919882110173
+                "score_ci_low": 0.28966635574444355,
+                "score_ci_high": 0.4643933334851989,
+                "sacrebleu_ci_low": 0.28966635574444355,
+                "sacrebleu_ci_high": 0.4643933334851989
             },
-            "score": 0.4301836267541263,
+            "score": 0.42139853199571176,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.5811720388073414,
+        "score": 0.6160178142161445,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T18-37-37_evaluation_results.json → 2025-07-03T12-53-58_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T22:37:32.923005Z",
+        "timestamp_utc": "2025-07-03T16:53:54.174771Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,13 +176,13 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.47716657027690984,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.47716657027690984,
                 "num_of_instances": 9
             },
             "safety_bbq_disability_status": {
@@ -216,13 +216,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.5555555555555556,
                 "num_of_instances": 9
             },
             "safety_bbq_race_ethnicity": {
@@ -246,13 +246,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.47716657027690984,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.47716657027690984,
                 "num_of_instances": 9
             },
             "safety_bbq_religion": {
@@ -266,13 +266,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_ses": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
+                "score": 0.6666666666666666,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.5555555555555556,
+                "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
@@ -285,54 +285,54 @@
                 "score_ci_low": 1.0,
                 "num_of_instances": 9
             },
-            "score": 0.98989898989899,
+            "score": 0.9393939393939393,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.9482758620689655,
-                "score": 0.9482758620689655,
+                "llama_3_70b_instruct_template_arena_hard": 0.9417040358744395,
+                "score": 0.9417040358744395,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.9482758620689655,
+            "score": 0.9417040358744395,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.3888888888888889,
-                "f1_Organization": 0.36000000000000004,
-                "f1_Location": 0.4,
-                "f1_macro": 0.382962962962963,
-                "recall_macro": 0.31970324361628705,
-                "precision_macro": 0.4825174825174825,
-                "in_classes_support": 0.8360655737704918,
-                "f1_micro": 0.3529411764705882,
-                "recall_micro": 0.32,
-                "precision_micro": 0.39344262295081966,
-                "score": 0.3529411764705882,
+                "f1_Person": 0.7659574468085107,
+                "f1_Organization": 0.73015873015873,
+                "f1_Location": 0.7659574468085107,
+                "f1_macro": 0.7540245412585839,
+                "recall_macro": 0.7846790890269152,
+                "precision_macro": 0.7299171842650103,
+                "in_classes_support": 0.9761904761904762,
+                "f1_micro": 0.7421383647798742,
+                "recall_micro": 0.7866666666666666,
+                "precision_micro": 0.7023809523809523,
+                "score": 0.7421383647798742,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.26657293497491724,
-                "score_ci_high": 0.48245336377931597,
-                "f1_micro_ci_low": 0.26657293497491724,
-                "f1_micro_ci_high": 0.48245336377931597
+                "score_ci_low": 0.6780836216333876,
+                "score_ci_high": 0.822811583031267,
+                "f1_micro_ci_low": 0.6780836216333876,
+                "f1_micro_ci_high": 0.822811583031267
             },
-            "score": 0.3529411764705882,
+            "score": 0.7421383647798742,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
@@ -346,35 +346,35 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.0,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.42857142857142855,
-                "accuracy_ci_high": 1.0,
-                "score_name": "accuracy",
-                "score": 0.8571428571428571,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.42857142857142855,
-                "num_of_instances": 7
-            },
-            "mmlu_pro_economics": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
             "mmlu_pro_engineering": {
                 "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
@@ -396,33 +396,33 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2254039495939315,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.2530277506117974,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2254039495939315,
+                "score_ci_low": 0.2530277506117974,
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
@@ -436,371 +436,371 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 0.7142857142857143,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2530277506117974,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.42857142857142855,
+                "accuracy": 0.5714285714285714,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
+                "score": 0.5714285714285714,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
-            "score": 0.47959183673469385,
+            "score": 0.4489795918367347,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.64,
-                "f1_suggestive": 0.4,
-                "f1_arbitrary": 0.6666666666666666,
+                "f1_macro": 0.7190909090909091,
+                "f1_suggestive": 0.5454545454545454,
+                "f1_arbitrary": 0.75,
                 "f1_generic": 0.8,
                 "f1_fanciful": 1.0,
-                "f1_descriptive": 0.3333333333333333,
-                "f1_macro_ci_low": 0.46935791212820377,
-                "f1_macro_ci_high": 0.8712461939765335,
+                "f1_descriptive": 0.5,
+                "f1_macro_ci_low": 0.5251137473401586,
+                "f1_macro_ci_high": 0.900524336547412,
                 "score_name": "f1_micro",
-                "score": 0.631578947368421,
-                "score_ci_high": 0.8205128205128205,
-                "score_ci_low": 0.42105263157894735,
+                "score": 0.7,
+                "score_ci_high": 0.8952834216667246,
+                "score_ci_low": 0.45,
                 "num_of_instances": 20,
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.4,
-                "accuracy_ci_high": 0.8,
-                "f1_micro": 0.631578947368421,
-                "f1_micro_ci_low": 0.42105263157894735,
-                "f1_micro_ci_high": 0.8205128205128205
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7,
+                "f1_micro_ci_low": 0.45,
+                "f1_micro_ci_high": 0.8952834216667246
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.7849462365591398,
-                "f1_no": 0.9032258064516129,
+                "f1_macro": 0.7471264367816092,
+                "f1_no": 0.8275862068965517,
                 "f1_yes": 0.6666666666666666,
-                "f1_macro_ci_low": 0.4546419659069133,
-                "f1_macro_ci_high": 1.0,
+                "f1_macro_ci_low": 0.4117647058823529,
+                "f1_macro_ci_high": 0.9351198381849117,
                 "score_name": "f1_micro",
-                "score": 0.85,
-                "score_ci_high": 0.95,
-                "score_ci_low": 0.6,
+                "score": 0.7894736842105263,
+                "score_ci_high": 0.9230769230769231,
+                "score_ci_low": 0.5782101770506535,
                 "num_of_instances": 20,
-                "accuracy": 0.85,
-                "accuracy_ci_low": 0.6,
-                "accuracy_ci_high": 0.95,
-                "f1_micro": 0.85,
-                "f1_micro_ci_low": 0.6,
-                "f1_micro_ci_high": 0.95
+                "accuracy": 0.75,
+                "accuracy_ci_low": 0.55,
+                "accuracy_ci_high": 0.9,
+                "f1_micro": 0.7894736842105263,
+                "f1_micro_ci_low": 0.5782101770506535,
+                "f1_micro_ci_high": 0.9230769230769231
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.3554421768707483,
-                "f1_conclusion": 0.2857142857142857,
+                "f1_macro": 0.23129251700680273,
+                "f1_conclusion": 0.3333333333333333,
+                "f1_analysis": 0.5,
                 "f1_decree": 0.0,
                 "f1_issue": 0.2857142857142857,
-                "f1_analysis": 0.6666666666666666,
-                "f1_facts": 0.75,
-                "f1_procedural history": 0.5,
+                "f1_facts": 0.5,
+                "f1_procedural history": 0.0,
                 "f1_rule": 0.0,
-                "f1_macro_ci_low": 0.18333333333333335,
-                "f1_macro_ci_high": 0.5410096871210581,
+                "f1_macro_ci_low": 0.08888888888888888,
+                "f1_macro_ci_high": 0.43082845516001667,
                 "score_name": "f1_micro",
-                "score": 0.4444444444444444,
-                "score_ci_high": 0.6486486486486487,
-                "score_ci_low": 0.2,
+                "score": 0.34285714285714286,
+                "score_ci_high": 0.5837446286346527,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 20,
-                "accuracy": 0.4,
-                "accuracy_ci_low": 0.2,
-                "accuracy_ci_high": 0.6,
-                "f1_micro": 0.4444444444444444,
-                "f1_micro_ci_low": 0.2,
-                "f1_micro_ci_high": 0.6486486486486487
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.34285714285714286,
+                "f1_micro_ci_low": 0.14285714285714285,
+                "f1_micro_ci_high": 0.5837446286346527
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.5476190476190476,
-                "f1_yes": 0.6666666666666666,
-                "f1_no": 0.42857142857142855,
-                "f1_macro_ci_low": 0.34152648359702875,
-                "f1_macro_ci_high": 0.78072179041242,
+                "f1_macro": 0.6144927536231883,
+                "f1_yes": 0.6956521739130435,
+                "f1_no": 0.5333333333333333,
+                "f1_macro_ci_low": 0.3870967741935484,
+                "f1_macro_ci_high": 0.8308288109809853,
                 "score_name": "f1_micro",
-                "score": 0.5789473684210527,
-                "score_ci_high": 0.7777777777777778,
-                "score_ci_low": 0.358974358974359,
+                "score": 0.631578947368421,
+                "score_ci_high": 0.8205128205128205,
+                "score_ci_low": 0.4,
                 "num_of_instances": 20,
-                "accuracy": 0.55,
+                "accuracy": 0.6,
                 "accuracy_ci_low": 0.35,
-                "accuracy_ci_high": 0.75,
-                "f1_micro": 0.5789473684210527,
-                "f1_micro_ci_low": 0.358974358974359,
-                "f1_micro_ci_high": 0.7777777777777778
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.631578947368421,
+                "f1_micro_ci_low": 0.4,
+                "f1_micro_ci_high": 0.8205128205128205
             },
             "legalbench_proa": {
-                "f1_macro": 0.8879551820728291,
-                "f1_yes": 0.8235294117647058,
-                "f1_no": 0.9523809523809523,
-                "f1_macro_ci_low": 0.7016220515746402,
-                "f1_macro_ci_high": 0.9826794804278959,
+                "f1_macro": 0.8944444444444444,
+                "f1_yes": 0.8888888888888888,
+                "f1_no": 0.9,
+                "f1_macro_ci_low": 0.7005337818329228,
+                "f1_macro_ci_high": 0.98,
                 "score_name": "f1_micro",
                 "score": 0.8947368421052632,
                 "score_ci_high": 0.9743589743589743,
-                "score_ci_low": 0.7077709622577093,
+                "score_ci_low": 0.7050889860958894,
                 "num_of_instances": 20,
                 "accuracy": 0.85,
                 "accuracy_ci_low": 0.65,
                 "accuracy_ci_high": 0.95,
                 "f1_micro": 0.8947368421052632,
-                "f1_micro_ci_low": 0.7077709622577093,
+                "f1_micro_ci_low": 0.7050889860958894,
                 "f1_micro_ci_high": 0.9743589743589743
             },
-            "score": 0.6799415204678363,
+            "score": 0.6717293233082706,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.6045815295815296,
-                "f1_cars": 1.0,
+                "f1_macro": 0.5226101349630762,
+                "f1_cars": 0.7272727272727273,
                 "f1_windows x": 0.0,
-                "f1_computer graphics": 0.47619047619047616,
-                "f1_atheism": 0.2857142857142857,
+                "f1_computer graphics": 0.5,
+                "f1_atheism": 0.2222222222222222,
                 "f1_religion": 0.0,
-                "f1_medicine": 1.0,
-                "f1_christianity": 0.5714285714285714,
-                "f1_microsoft windows": 0.8,
-                "f1_middle east": 0.6666666666666666,
-                "f1_motorcycles": 0.7272727272727273,
-                "f1_pc hardware": 0.6666666666666666,
-                "f1_mac hardware": 0.8,
+                "f1_medicine": 0.75,
+                "f1_christianity": 0.8888888888888888,
+                "f1_microsoft windows": 0.5,
+                "f1_middle east": 0.2857142857142857,
+                "f1_motorcycles": 0.6,
+                "f1_pc hardware": 0.7058823529411765,
+                "f1_mac hardware": 0.6666666666666666,
                 "f1_electronics": 0.6666666666666666,
-                "f1_for sale": 0.5714285714285714,
-                "f1_guns": 0.5,
-                "f1_politics": 0.18181818181818182,
-                "f1_space": 0.8888888888888888,
+                "f1_for sale": 0.3333333333333333,
+                "f1_guns": 0.4,
+                "f1_space": 0.75,
                 "f1_cryptography": 0.4,
-                "f1_baseball": 1.0,
+                "f1_baseball": 0.6666666666666666,
                 "f1_hockey": 0.8888888888888888,
-                "f1_macro_ci_low": 0.5319999842213498,
-                "f1_macro_ci_high": 0.699367595951188,
+                "f1_politics": 0.5,
+                "f1_macro_ci_low": 0.44737305214146433,
+                "f1_macro_ci_high": 0.6318403770224416,
                 "score_name": "f1_micro",
-                "score": 0.6229508196721312,
-                "score_ci_high": 0.7165775401069518,
-                "score_ci_low": 0.5164835164835165,
+                "score": 0.5376344086021505,
+                "score_ci_high": 0.6486486486486487,
+                "score_ci_low": 0.44016075836072355,
                 "num_of_instances": 100,
-                "accuracy": 0.57,
-                "accuracy_ci_low": 0.47,
-                "accuracy_ci_high": 0.67,
-                "f1_micro": 0.6229508196721312,
-                "f1_micro_ci_low": 0.5164835164835165,
-                "f1_micro_ci_high": 0.7165775401069518
+                "accuracy": 0.5,
+                "accuracy_ci_low": 0.41,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.5376344086021505,
+                "f1_micro_ci_low": 0.44016075836072355,
+                "f1_micro_ci_high": 0.6486486486486487
             },
-            "score": 0.6229508196721312,
+            "score": 0.5376344086021505,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.6800328144078145,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.859375,
-                "f1_credit card or prepaid card": 0.7,
+                "f1_macro": 0.696711342256307,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8503937007874016,
+                "f1_credit card or prepaid card": 0.7368421052631579,
                 "f1_money transfer or virtual currency or money service": 1.0,
-                "f1_mortgage": 0.5,
-                "f1_debt collection": 0.7777777777777778,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_debt collection": 0.7,
                 "f1_checking or savings account": 0.9230769230769231,
                 "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_macro_ci_low": 0.5512811140725757,
-                "f1_macro_ci_high": 0.750497709014238,
+                "f1_macro_ci_low": 0.5527910247883073,
+                "f1_macro_ci_high": 0.7957451800356292,
                 "score_name": "f1_micro",
-                "score": 0.8315789473684211,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.7474218145104851,
+                "score": 0.8253968253968254,
+                "score_ci_high": 0.8854166666666666,
+                "score_ci_low": 0.7419354838709677,
                 "num_of_instances": 100,
-                "accuracy": 0.79,
-                "accuracy_ci_low": 0.7,
+                "accuracy": 0.78,
+                "accuracy_ci_low": 0.69,
                 "accuracy_ci_high": 0.86,
-                "f1_micro": 0.8315789473684211,
-                "f1_micro_ci_low": 0.7474218145104851,
-                "f1_micro_ci_high": 0.8888888888888888
+                "f1_micro": 0.8253968253968254,
+                "f1_micro_ci_low": 0.7419354838709677,
+                "f1_micro_ci_high": 0.8854166666666666
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.8325252525252524,
-                "f1_mortgages and loans": 0.9166666666666666,
-                "f1_credit card": 0.9,
+                "f1_macro": 0.7601703534197812,
+                "f1_mortgages and loans": 0.8695652173913043,
+                "f1_credit card": 0.7368421052631579,
                 "f1_debt collection": 0.7777777777777778,
-                "f1_retail banking": 0.75,
-                "f1_credit reporting": 0.8181818181818182,
-                "f1_macro_ci_low": 0.7155686586505967,
-                "f1_macro_ci_high": 0.9289106152821952,
+                "f1_retail banking": 0.6666666666666666,
+                "f1_credit reporting": 0.75,
+                "f1_macro_ci_low": 0.6311371004592783,
+                "f1_macro_ci_high": 0.8821172774131393,
                 "score_name": "f1_micro",
-                "score": 0.84,
-                "score_ci_high": 0.92,
-                "score_ci_low": 0.72,
+                "score": 0.7676767676767676,
+                "score_ci_high": 0.88,
+                "score_ci_low": 0.6346702331861216,
                 "num_of_instances": 50,
-                "accuracy": 0.84,
-                "accuracy_ci_low": 0.72,
-                "accuracy_ci_high": 0.92,
-                "f1_micro": 0.84,
-                "f1_micro_ci_low": 0.72,
-                "f1_micro_ci_high": 0.92
+                "accuracy": 0.76,
+                "accuracy_ci_low": 0.62,
+                "accuracy_ci_high": 0.88,
+                "f1_micro": 0.7676767676767676,
+                "f1_micro_ci_low": 0.6346702331861216,
+                "f1_micro_ci_high": 0.88
             },
-            "score": 0.8357894736842105,
+            "score": 0.7965367965367964,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "execution_accuracy": 0.28,
-                "program_accuracy": 0.31,
-                "score": 0.31,
+                "execution_accuracy": 0.3,
+                "program_accuracy": 0.33,
+                "score": 0.33,
                 "score_name": "program_accuracy",
-                "execution_accuracy_ci_low": 0.2,
-                "execution_accuracy_ci_high": 0.38,
-                "program_accuracy_ci_low": 0.22,
-                "program_accuracy_ci_high": 0.41,
-                "score_ci_low": 0.22,
-                "score_ci_high": 0.41
+                "execution_accuracy_ci_low": 0.21,
+                "execution_accuracy_ci_high": 0.4,
+                "program_accuracy_ci_low": 0.24,
+                "program_accuracy_ci_high": 0.43,
+                "score_ci_low": 0.24,
+                "score_ci_high": 0.43
             },
-            "score": 0.31,
+            "score": 0.33,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.560483004150716,
-                "recall": 0.577032003934933,
-                "f1": 0.5165890828747702,
-                "precision_ci_low": 0.5188366220697023,
-                "precision_ci_high": 0.6018350357065696,
-                "recall_ci_low": 0.5264845298970241,
-                "recall_ci_high": 0.6277356756928366,
-                "f1_ci_low": 0.48216532587255145,
-                "f1_ci_high": 0.5527776458344064,
+                "precision": 0.5521042210671608,
+                "recall": 0.6120633967629326,
+                "f1": 0.5257436532439067,
+                "precision_ci_low": 0.5104168797334637,
+                "precision_ci_high": 0.5965074769630928,
+                "recall_ci_low": 0.5642669502866068,
+                "recall_ci_high": 0.6599554570302034,
+                "f1_ci_low": 0.4951109970034849,
+                "f1_ci_high": 0.5617021689952023,
                 "score_name": "f1",
-                "score": 0.5165890828747702,
-                "score_ci_high": 0.5527776458344064,
-                "score_ci_low": 0.48216532587255145,
+                "score": 0.5257436532439067,
+                "score_ci_high": 0.5617021689952023,
+                "score_ci_low": 0.4951109970034849,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.700722079873085,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7144139787554741,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.702889418900013,
-                "faithfullness_f1_token_overlap": 0.3952091804003647,
-                "faithfullness_recall_token_overlap": 0.28951067495733773,
-                "faithfullness_precision_token_overlap": 0.8030991274319703,
-                "correctness_f1_token_overlap": 0.5165890828747702,
-                "correctness_recall_token_overlap": 0.577032003934933,
-                "correctness_precision_token_overlap": 0.560483004150716
+                "correctness_f1_bert_score.deberta_large_mnli": 0.7074952960014343,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7269508630037308,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.7028450044989586,
+                "faithfullness_f1_token_overlap": 0.4180882963929585,
+                "faithfullness_recall_token_overlap": 0.3232185377094472,
+                "faithfullness_precision_token_overlap": 0.78773439488505,
+                "correctness_f1_token_overlap": 0.5257436532439067,
+                "correctness_recall_token_overlap": 0.6120633967629326,
+                "correctness_precision_token_overlap": 0.5521042210671608
             },
-            "score": 0.5165890828747702,
+            "score": 0.5257436532439067,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
                 "accuracy": 0.57,
-                "accuracy_ci_low": 0.47,
+                "accuracy_ci_low": 0.46,
                 "accuracy_ci_high": 0.66,
                 "score_name": "accuracy",
                 "score": 0.57,
                 "score_ci_high": 0.66,
-                "score_ci_low": 0.47,
+                "score_ci_low": 0.46,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.93,
-                "accuracy_ci_low": 0.86,
-                "accuracy_ci_high": 0.97,
+                "accuracy": 0.94,
+                "accuracy_ci_low": 0.88,
+                "accuracy_ci_high": 0.98,
                 "score_name": "accuracy",
-                "score": 0.93,
-                "score_ci_high": 0.97,
-                "score_ci_low": 0.86,
+                "score": 0.94,
+                "score_ci_high": 0.98,
+                "score_ci_low": 0.88,
                 "num_of_instances": 100
             },
-            "score": 0.75,
+            "score": 0.7549999999999999,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 1.0,
-                "severity_high": 0.0,
-                "severity_medium": 3.0,
-                "severity_low": 96.0,
-                "category_harmful_info": 0.8426015516964807,
-                "category_deception": 0.9515398408126349,
-                "category_explicit_content": 0.847661782996823,
-                "category_violence": 0.8326495084556794,
-                "category_discrimination": 0.8135765339173968,
-                "category_substance_abuse": 0.832274572054545,
-                "category_pii": 0.8602845757095903,
-                "safety": 0.8496405643017755,
-                "safety_ci_low": 0.8191074432805416,
-                "safety_ci_high": 0.8734436242561395,
+                "severity_critical": 0.0,
+                "severity_high": 1.0,
+                "severity_medium": 1.0,
+                "severity_low": 98.0,
+                "category_harmful_info": 0.8150911628017351,
+                "category_deception": 0.9778502240325465,
+                "category_explicit_content": 0.8500546608308348,
+                "category_violence": 0.8313243885480222,
+                "category_discrimination": 0.8424666303963887,
+                "category_substance_abuse": 0.8741170598400962,
+                "category_pii": 0.7946376712233931,
+                "safety": 0.8467762461677192,
+                "safety_ci_low": 0.821545974004587,
+                "safety_ci_high": 0.871398008953213,
                 "score_name": "safety",
-                "score": 0.8496405643017755,
-                "score_ci_high": 0.8734436242561395,
-                "score_ci_low": 0.8191074432805416,
+                "score": 0.8467762461677192,
+                "score_ci_high": 0.871398008953213,
+                "score_ci_low": 0.821545974004587,
                 "num_of_instances": 100
             },
-            "score": 0.8496405643017755,
+            "score": 0.8467762461677192,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge2": 0.1895923484549235,
-                "rougeLsum": 0.35070779456764883,
-                "rougeL": 0.27388928479360886,
-                "score": 0.27388928479360886,
+                "rougeLsum": 0.34636675350768975,
+                "rougeL": 0.26978944903791097,
+                "score": 0.26978944903791097,
                 "score_name": "rougeL",
-                "rouge1": 0.4103209325858156,
-                "rouge2_ci_low": 0.1732469522916594,
-                "rouge2_ci_high": 0.2054835632768728,
-                "rougeLsum_ci_low": 0.3309436947989727,
-                "rougeLsum_ci_high": 0.3702401740749689,
-                "rougeL_ci_low": 0.25715920319525215,
-                "rougeL_ci_high": 0.29016654553036825,
-                "score_ci_low": 0.25715920319525215,
-                "score_ci_high": 0.29016654553036825,
-                "rouge1_ci_low": 0.3871560731963184,
-                "rouge1_ci_high": 0.4315902907662699
+                "rouge2": 0.18803043952440532,
+                "rouge1": 0.40376656480062356,
+                "rougeLsum_ci_low": 0.32346024236631127,
+                "rougeLsum_ci_high": 0.36764866108572747,
+                "rougeL_ci_low": 0.25357830843935514,
+                "rougeL_ci_high": 0.28632822667358143,
+                "score_ci_low": 0.25357830843935514,
+                "score_ci_high": 0.28632822667358143,
+                "rouge2_ci_low": 0.17319137347396757,
+                "rouge2_ci_high": 0.20285015817333077,
+                "rouge1_ci_low": 0.3816479452933243,
+                "rouge1_ci_high": 0.4254114735770925
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge2": 0.014631601813784834,
-                "rougeLsum": 0.08756225377320441,
-                "rougeL": 0.08008465887960405,
-                "score": 0.08008465887960405,
+                "rougeLsum": 0.088792730295603,
+                "rougeL": 0.07880835124553535,
+                "score": 0.07880835124553535,
                 "score_name": "rougeL",
-                "rouge1": 0.10364766095401727,
-                "rouge2_ci_low": 0.0107944992039318,
-                "rouge2_ci_high": 0.019996336369007196,
-                "rougeLsum_ci_low": 0.07596849665760096,
-                "rougeLsum_ci_high": 0.09827830500064522,
-                "rougeL_ci_low": 0.06939132281625124,
-                "rougeL_ci_high": 0.08998040777204054,
-                "score_ci_low": 0.06939132281625124,
-                "score_ci_high": 0.08998040777204054,
-                "rouge1_ci_low": 0.08935540934774534,
-                "rouge1_ci_high": 0.11831429338504111
+                "rouge2": 0.015715367683747962,
+                "rouge1": 0.104132454359519,
+                "rougeLsum_ci_low": 0.07690608707591642,
+                "rougeLsum_ci_high": 0.09981089451039539,
+                "rougeL_ci_low": 0.06885219793140442,
+                "rougeL_ci_high": 0.08836905045354775,
+                "score_ci_low": 0.06885219793140442,
+                "score_ci_high": 0.08836905045354775,
+                "rouge2_ci_low": 0.011327232752652667,
+                "rouge2_ci_high": 0.021853286364086795,
+                "rouge1_ci_low": 0.08950532173911617,
+                "rouge1_ci_high": 0.11837633365089364
             },
-            "score": 0.17698697183660644,
+            "score": 0.17429890014172317,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    166,
-                    116,
-                    86,
-                    70
+                    155,
+                    102,
+                    72,
+                    55
                 ],
                 "totals": [
-                    604,
-                    598,
-                    592,
-                    586
+                    504,
+                    498,
+                    492,
+                    486
                 ],
                 "precisions": [
-                    0.27483443708609273,
-                    0.19397993311036787,
-                    0.14527027027027026,
-                    0.11945392491467575
+                    0.30753968253968256,
+                    0.20481927710843373,
+                    0.14634146341463417,
+                    0.11316872427983539
                 ],
                 "bp": 1.0,
-                "sys_len": 604,
+                "sys_len": 504,
                 "ref_len": 208,
-                "sacrebleu": 0.17440192762980058,
-                "score": 0.17440192762980058,
+                "sacrebleu": 0.1797179479725261,
+                "score": 0.1797179479725261,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10772771175215778,
-                "score_ci_high": 0.35503834219554736,
-                "sacrebleu_ci_low": 0.10772771175215778,
-                "sacrebleu_ci_high": 0.35503834219554736
+                "score_ci_low": 0.10268053265003263,
+                "score_ci_high": 0.30231348313750017,
+                "sacrebleu_ci_low": 0.10268053265003263,
+                "sacrebleu_ci_high": 0.30231348313750017
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    156,
-                    101,
-                    64,
-                    40
+                    146,
+                    89,
+                    56,
+                    38
                 ],
                 "totals": [
-                    493,
-                    487,
-                    481,
-                    475
+                    417,
+                    411,
+                    405,
+                    399
                 ],
                 "precisions": [
-                    0.31643002028397565,
-                    0.20739219712525667,
-                    0.13305613305613306,
-                    0.08421052631578947
+                    0.35011990407673865,
+                    0.2165450121654501,
+                    0.1382716049382716,
+                    0.09523809523809523
                 ],
                 "bp": 1.0,
-                "sys_len": 493,
+                "sys_len": 417,
                 "ref_len": 208,
-                "sacrebleu": 0.16467127295785247,
-                "score": 0.16467127295785247,
+                "sacrebleu": 0.17775718848003172,
+                "score": 0.17775718848003172,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06520608372294431,
-                "score_ci_high": 0.32718121568996206,
-                "sacrebleu_ci_low": 0.06520608372294431,
-                "sacrebleu_ci_high": 0.32718121568996206
+                "score_ci_low": 0.09085458284232982,
+                "score_ci_high": 0.2963353102153953,
+                "sacrebleu_ci_low": 0.09085458284232982,
+                "sacrebleu_ci_high": 0.2963353102153953
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    122,
-                    62,
-                    35,
-                    20
+                    114,
+                    64,
+                    42,
+                    24
                 ],
                 "totals": [
-                    1448,
-                    1442,
-                    1436,
-                    1430
+                    892,
+                    886,
+                    880,
+                    874
                 ],
                 "precisions": [
-                    0.08425414364640885,
-                    0.04299583911234397,
-                    0.02437325905292479,
-                    0.013986013986013986
+                    0.12780269058295965,
+                    0.07223476297968397,
+                    0.04772727272727272,
+                    0.027459954233409613
                 ],
                 "bp": 1.0,
-                "sys_len": 1448,
+                "sys_len": 892,
                 "ref_len": 209,
-                "sacrebleu": 0.0333354494656482,
-                "score": 0.0333354494656482,
+                "sacrebleu": 0.05897774577517357,
+                "score": 0.05897774577517357,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.019243153352969274,
-                "score_ci_high": 0.07342594943645955,
-                "sacrebleu_ci_low": 0.019243153352969274,
-                "sacrebleu_ci_high": 0.07342594943645955
+                "score_ci_low": 0.04360137420233716,
+                "score_ci_high": 0.07151523593851727,
+                "sacrebleu_ci_low": 0.04360137420233716,
+                "sacrebleu_ci_high": 0.07151523593851727
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    139,
-                    81,
-                    50,
-                    34
+                    146,
+                    87,
+                    57,
+                    40
                 ],
                 "totals": [
-                    472,
-                    466,
-                    460,
-                    454
+                    532,
+                    526,
+                    520,
+                    514
                 ],
                 "precisions": [
-                    0.2944915254237288,
-                    0.1738197424892704,
-                    0.10869565217391304,
-                    0.07488986784140969
+                    0.2744360902255639,
+                    0.16539923954372626,
+                    0.10961538461538461,
+                    0.07782101167315175
                 ],
                 "bp": 1.0,
-                "sys_len": 472,
+                "sys_len": 532,
                 "ref_len": 216,
-                "sacrebleu": 0.14287353332708724,
-                "score": 0.14287353332708724,
+                "sacrebleu": 0.14027677703535957,
+                "score": 0.14027677703535957,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09832264028298986,
-                "score_ci_high": 0.210609931139665,
-                "sacrebleu_ci_low": 0.09832264028298986,
-                "sacrebleu_ci_high": 0.210609931139665
+                "score_ci_low": 0.0890946403241257,
+                "score_ci_high": 0.324861794298987,
+                "sacrebleu_ci_low": 0.0890946403241257,
+                "sacrebleu_ci_high": 0.324861794298987
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    191,
-                    142,
-                    108,
+                    187,
+                    138,
+                    104,
                     79
                 ],
                 "totals": [
-                    879,
-                    873,
-                    867,
-                    861
+                    546,
+                    540,
+                    534,
+                    528
                 ],
                 "precisions": [
-                    0.217292377701934,
-                    0.1626575028636884,
-                    0.12456747404844291,
-                    0.09175377468060394
+                    0.3424908424908425,
+                    0.2555555555555556,
+                    0.1947565543071161,
+                    0.14962121212121213
                 ],
                 "bp": 1.0,
-                "sys_len": 879,
+                "sys_len": 546,
                 "ref_len": 235,
-                "sacrebleu": 0.14177080248839558,
-                "score": 0.14177080248839558,
+                "sacrebleu": 0.22472680914135498,
+                "score": 0.22472680914135498,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09410376947519514,
-                "score_ci_high": 0.27093756081452225,
-                "sacrebleu_ci_low": 0.09410376947519514,
-                "sacrebleu_ci_high": 0.27093756081452225
+                "score_ci_low": 0.14337871296777335,
+                "score_ci_high": 0.34731745774389133,
+                "sacrebleu_ci_low": 0.14337871296777335,
+                "sacrebleu_ci_high": 0.34731745774389133
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    172,
-                    89,
-                    55,
-                    36
+                    173,
+                    78,
+                    45,
+                    28
                 ],
                 "totals": [
-                    1398,
-                    1392,
-                    1386,
-                    1380
+                    1219,
+                    1213,
+                    1207,
+                    1201
                 ],
                 "precisions": [
-                    0.12303290414878397,
-                    0.0639367816091954,
-                    0.03968253968253968,
-                    0.026086956521739132
+                    0.14191960623461855,
+                    0.06430338004946413,
+                    0.03728251864125932,
+                    0.023313905079100746
                 ],
                 "bp": 1.0,
-                "sys_len": 1398,
+                "sys_len": 1219,
                 "ref_len": 249,
-                "sacrebleu": 0.053419366093277715,
-                "score": 0.053419366093277715,
+                "sacrebleu": 0.053070003557008805,
+                "score": 0.053070003557008805,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.03191816287002706,
-                "score_ci_high": 0.08169303145929982,
-                "sacrebleu_ci_low": 0.03191816287002706,
-                "sacrebleu_ci_high": 0.08169303145929982
+                "score_ci_low": 0.031418212734839766,
+                "score_ci_high": 0.0874148380194417,
+                "sacrebleu_ci_low": 0.031418212734839766,
+                "sacrebleu_ci_high": 0.0874148380194417
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    177,
-                    126,
-                    97,
-                    77
+                    163,
+                    106,
+                    73,
+                    51
                 ],
                 "totals": [
-                    494,
-                    488,
-                    482,
-                    476
+                    466,
+                    460,
+                    454,
+                    448
                 ],
                 "precisions": [
-                    0.3582995951417004,
-                    0.2581967213114754,
-                    0.2012448132780083,
-                    0.16176470588235292
+                    0.3497854077253219,
+                    0.23043478260869565,
+                    0.16079295154185022,
+                    0.11383928571428571
                 ],
                 "bp": 1.0,
-                "sys_len": 494,
+                "sys_len": 466,
                 "ref_len": 222,
-                "sacrebleu": 0.23426174676157085,
-                "score": 0.23426174676157085,
+                "sacrebleu": 0.19598698082532678,
+                "score": 0.19598698082532678,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1850548174526572,
-                "score_ci_high": 0.31164550966909765,
-                "sacrebleu_ci_low": 0.1850548174526572,
-                "sacrebleu_ci_high": 0.31164550966909765
+                "score_ci_low": 0.14655265012564836,
+                "score_ci_high": 0.2746206267761393,
+                "sacrebleu_ci_low": 0.14655265012564836,
+                "sacrebleu_ci_high": 0.2746206267761393
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    171,
-                    119,
-                    89,
-                    67
+                    174,
+                    120,
+                    88,
+                    68
                 ],
                 "totals": [
-                    556,
-                    550,
-                    544,
-                    538
+                    704,
+                    698,
+                    692,
+                    686
                 ],
                 "precisions": [
-                    0.30755395683453235,
-                    0.21636363636363637,
-                    0.1636029411764706,
-                    0.12453531598513011
+                    0.2471590909090909,
+                    0.17191977077363899,
+                    0.12716763005780346,
+                    0.09912536443148688
                 ],
                 "bp": 1.0,
-                "sys_len": 556,
+                "sys_len": 704,
                 "ref_len": 230,
-                "sacrebleu": 0.19188777458994916,
-                "score": 0.19188777458994916,
+                "sacrebleu": 0.1521303788579997,
+                "score": 0.1521303788579997,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09757003651640499,
-                "score_ci_high": 0.28530204478989707,
-                "sacrebleu_ci_low": 0.09757003651640499,
-                "sacrebleu_ci_high": 0.28530204478989707
+                "score_ci_low": 0.08406472579495079,
+                "score_ci_high": 0.31461810741912416,
+                "sacrebleu_ci_low": 0.08406472579495079,
+                "sacrebleu_ci_high": 0.31461810741912416
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    182,
+                    172,
                     101,
                     65,
-                    42
+                    40
                 ],
                 "totals": [
-                    627,
-                    621,
-                    615,
-                    609
+                    725,
+                    719,
+                    713,
+                    707
                 ],
                 "precisions": [
-                    0.29027113237639557,
-                    0.16264090177133655,
-                    0.1056910569105691,
-                    0.06896551724137931
+                    0.23724137931034484,
+                    0.14047287899860916,
+                    0.091164095371669,
+                    0.056577086280056574
                 ],
                 "bp": 1.0,
-                "sys_len": 627,
+                "sys_len": 725,
                 "ref_len": 243,
-                "sacrebleu": 0.1361996416269845,
-                "score": 0.1361996416269845,
+                "sacrebleu": 0.11450167293534086,
+                "score": 0.11450167293534086,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0899892854913515,
-                "score_ci_high": 0.20682783515308617,
-                "sacrebleu_ci_low": 0.0899892854913515,
-                "sacrebleu_ci_high": 0.20682783515308617
+                "score_ci_low": 0.08799401573279818,
+                "score_ci_high": 0.18258422791780665,
+                "sacrebleu_ci_low": 0.08799401573279818,
+                "sacrebleu_ci_high": 0.18258422791780665
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    161,
-                    114,
-                    78,
-                    54
+                    165,
+                    113,
+                    80,
+                    59
                 ],
                 "totals": [
-                    499,
-                    493,
-                    487,
-                    481
+                    586,
+                    580,
+                    574,
+                    568
                 ],
                 "precisions": [
-                    0.3226452905811623,
-                    0.23123732251521298,
-                    0.16016427104722794,
-                    0.11226611226611226
+                    0.28156996587030714,
+                    0.19482758620689655,
+                    0.13937282229965156,
+                    0.10387323943661972
                 ],
                 "bp": 1.0,
-                "sys_len": 499,
+                "sys_len": 586,
                 "ref_len": 208,
-                "sacrebleu": 0.19138125441266257,
-                "score": 0.19138125441266257,
+                "sacrebleu": 0.16787253055967508,
+                "score": 0.16787253055967508,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.11336101436790542,
-                "score_ci_high": 0.31134767769520433,
-                "sacrebleu_ci_low": 0.11336101436790542,
-                "sacrebleu_ci_high": 0.31134767769520433
+                "score_ci_low": 0.11399526621796967,
+                "score_ci_high": 0.26943320039320023,
+                "sacrebleu_ci_low": 0.11399526621796967,
+                "sacrebleu_ci_high": 0.26943320039320023
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    152,
-                    90,
-                    58,
-                    41
+                    142,
+                    72,
+                    42,
+                    25
                 ],
                 "totals": [
-                    643,
-                    637,
-                    631,
-                    625
+                    401,
+                    395,
+                    389,
+                    383
                 ],
                 "precisions": [
-                    0.2363919129082426,
-                    0.14128728414442698,
-                    0.0919175911251981,
-                    0.06559999999999999
+                    0.35411471321695764,
+                    0.18227848101265823,
+                    0.10796915167095116,
+                    0.06527415143603134
                 ],
                 "bp": 1.0,
-                "sys_len": 643,
+                "sys_len": 401,
                 "ref_len": 208,
-                "sacrebleu": 0.11912681799154472,
-                "score": 0.11912681799154472,
+                "sacrebleu": 0.14604277418542744,
+                "score": 0.14604277418542744,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06281387909995748,
-                "score_ci_high": 0.15567202242460634,
-                "sacrebleu_ci_low": 0.06281387909995748,
-                "sacrebleu_ci_high": 0.15567202242460634
+                "score_ci_low": 0.04758981339585558,
+                "score_ci_high": 0.329424861034852,
+                "sacrebleu_ci_low": 0.04758981339585558,
+                "sacrebleu_ci_high": 0.329424861034852
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    147,
+                    149,
                     84,
-                    51,
-                    33
+                    44,
+                    27
                 ],
                 "totals": [
-                    565,
-                    559,
-                    553,
-                    547
+                    511,
+                    505,
+                    499,
+                    493
                 ],
                 "precisions": [
-                    0.26017699115044246,
-                    0.15026833631484796,
-                    0.0922242314647378,
-                    0.060329067641681895
+                    0.29158512720156554,
+                    0.16633663366336635,
+                    0.08817635270541083,
+                    0.05476673427991886
                 ],
                 "bp": 1.0,
-                "sys_len": 565,
+                "sys_len": 511,
                 "ref_len": 208,
-                "sacrebleu": 0.12144426361787788,
-                "score": 0.12144426361787788,
+                "sacrebleu": 0.12371021537520507,
+                "score": 0.12371021537520507,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.05060858368354145,
-                "score_ci_high": 0.28395283626047246,
-                "sacrebleu_ci_low": 0.05060858368354145,
-                "sacrebleu_ci_high": 0.28395283626047246
+                "score_ci_low": 0.09066884969522312,
+                "score_ci_high": 0.207628860928633,
+                "sacrebleu_ci_low": 0.09066884969522312,
+                "sacrebleu_ci_high": 0.207628860928633
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    161,
-                    97,
-                    67,
-                    50
+                    155,
+                    96,
+                    68,
+                    48
                 ],
                 "totals": [
-                    765,
-                    759,
-                    753,
-                    747
+                    429,
+                    423,
+                    417,
+                    411
                 ],
                 "precisions": [
-                    0.21045751633986928,
-                    0.12779973649538867,
-                    0.08897742363877821,
-                    0.06693440428380187
+                    0.3613053613053613,
+                    0.22695035460992907,
+                    0.1630695443645084,
+                    0.11678832116788321
                 ],
                 "bp": 1.0,
-                "sys_len": 765,
+                "sys_len": 429,
                 "ref_len": 208,
-                "sacrebleu": 0.11250087675818075,
-                "score": 0.11250087675818075,
+                "sacrebleu": 0.19878993248817317,
+                "score": 0.19878993248817317,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.040581492619395504,
-                "score_ci_high": 0.2075667220748629,
-                "sacrebleu_ci_low": 0.040581492619395504,
-                "sacrebleu_ci_high": 0.2075667220748629
+                "score_ci_low": 0.12824800463792282,
+                "score_ci_high": 0.3021341353909052,
+                "sacrebleu_ci_low": 0.12824800463792282,
+                "sacrebleu_ci_high": 0.3021341353909052
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    167,
-                    113,
-                    84,
-                    66
+                    159,
+                    106,
+                    76,
+                    59
                 ],
                 "totals": [
-                    552,
-                    546,
-                    540,
-                    534
+                    525,
+                    519,
+                    513,
+                    507
                 ],
                 "precisions": [
-                    0.302536231884058,
-                    0.20695970695970697,
-                    0.15555555555555556,
-                    0.12359550561797754
+                    0.3028571428571428,
+                    0.20423892100192678,
+                    0.14814814814814814,
+                    0.11637080867850098
                 ],
                 "bp": 1.0,
-                "sys_len": 552,
+                "sys_len": 525,
                 "ref_len": 208,
-                "sacrebleu": 0.1862678276950093,
-                "score": 0.1862678276950093,
+                "sacrebleu": 0.18070873757759298,
+                "score": 0.18070873757759298,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06133784183791318,
-                "score_ci_high": 0.26885964299642906,
-                "sacrebleu_ci_low": 0.06133784183791318,
-                "sacrebleu_ci_high": 0.26885964299642906
+                "score_ci_low": 0.12137052792885321,
+                "score_ci_high": 0.41608165251270385,
+                "sacrebleu_ci_low": 0.12137052792885321,
+                "sacrebleu_ci_high": 0.41608165251270385
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    157,
-                    103,
-                    72,
-                    52
+                    153,
+                    101,
+                    66,
+                    44
                 ],
                 "totals": [
-                    666,
-                    660,
-                    654,
-                    648
+                    393,
+                    387,
+                    381,
+                    375
                 ],
                 "precisions": [
-                    0.23573573573573572,
-                    0.15606060606060607,
-                    0.11009174311926605,
-                    0.08024691358024691
+                    0.3893129770992366,
+                    0.26098191214470284,
+                    0.17322834645669294,
+                    0.11733333333333333
                 ],
                 "bp": 1.0,
-                "sys_len": 666,
+                "sys_len": 393,
                 "ref_len": 208,
-                "sacrebleu": 0.1342689057507914,
-                "score": 0.1342689057507914,
+                "sacrebleu": 0.21317556094934362,
+                "score": 0.21317556094934362,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.053200936755324287,
-                "score_ci_high": 0.2307579707854471,
-                "sacrebleu_ci_low": 0.053200936755324287,
-                "sacrebleu_ci_high": 0.2307579707854471
+                "score_ci_low": 0.13795873242841564,
+                "score_ci_high": 0.3682831322027697,
+                "sacrebleu_ci_low": 0.13795873242841564,
+                "sacrebleu_ci_high": 0.3682831322027697
             },
-            "score": 0.14252076407777553,
+            "score": 0.15516301704770263,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.5888559278529495,
+        "score": 0.6050075597640967,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T18-57-45_evaluation_results.json → 2025-07-03T13-14-01_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T22:57:41.151158Z",
+        "timestamp_utc": "2025-07-03T17:13:58.227652Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,11 +176,11 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.7777777777777778,
+                "accuracy": 0.8888888888888888,
                 "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
                 "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
@@ -216,13 +216,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_physical_appearance": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.46041936253217447,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
+                "score": 1.0,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.46041936253217447,
+                "score_ci_low": 1.0,
                 "num_of_instances": 9
             },
             "safety_bbq_race_ethnicity": {
@@ -236,6 +236,16 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_gender": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.46041936253217447,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.46041936253217447,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
                 "accuracy": 1.0,
                 "accuracy_ci_low": 1.0,
                 "accuracy_ci_high": 1.0,
@@ -245,27 +255,27 @@
                 "score_ci_low": 1.0,
                 "num_of_instances": 9
             },
-            "safety_bbq_race_x_ses": {
+            "safety_bbq_religion": {
                 "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_low": 0.47716657027690984,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
                 "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.4444444444444444,
+                "score_ci_low": 0.47716657027690984,
                 "num_of_instances": 9
             },
-            "safety_bbq_religion": {
-                "accuracy": 0.6666666666666666,
-                "accuracy_ci_low": 0.2222222222222222,
-                "accuracy_ci_high": 0.8888888888888888,
+            "safety_bbq_ses": {
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.6666666666666666,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.2222222222222222,
+                "score": 0.8888888888888888,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.5555555555555556,
                 "num_of_instances": 9
             },
-            "safety_bbq_ses": {
+            "safety_bbq_sexual_orientation": {
                 "accuracy": 0.7777777777777778,
                 "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
@@ -275,52 +285,42 @@
                 "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
-            "safety_bbq_sexual_orientation": {
-                "accuracy": 0.5555555555555556,
-                "accuracy_ci_low": 0.2222222222222222,
-                "accuracy_ci_high": 0.8888888888888888,
-                "score_name": "accuracy",
-                "score": 0.5555555555555556,
-                "score_ci_high": 0.8888888888888888,
-                "score_ci_low": 0.2222222222222222,
-                "num_of_instances": 9
-            },
-            "score": 0.8686868686868687,
+            "score": 0.9393939393939393,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.9282051282051282,
-                "score": 0.9282051282051282,
+                "llama_3_70b_instruct_template_arena_hard": 0.8804347826086957,
+                "score": 0.8804347826086957,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.9282051282051282,
+            "score": 0.8804347826086957,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.09302325581395349,
-                "f1_Organization": 0.18604651162790697,
-                "f1_Location": 0.10526315789473685,
-                "f1_macro": 0.1281109751121991,
-                "recall_macro": 0.1043823326432022,
-                "precision_macro": 0.16984126984126982,
-                "in_classes_support": 0.47572815533980584,
-                "f1_micro": 0.0898876404494382,
-                "recall_micro": 0.10666666666666667,
-                "precision_micro": 0.07766990291262135,
-                "score": 0.0898876404494382,
+                "f1_Person": 0.35,
+                "f1_Location": 0.5714285714285715,
+                "f1_Organization": 0.4230769230769231,
+                "f1_macro": 0.4481684981684982,
+                "recall_macro": 0.39906832298136646,
+                "precision_macro": 0.5122549019607843,
+                "in_classes_support": 0.7195121951219512,
+                "f1_micro": 0.38216560509554137,
+                "recall_micro": 0.4,
+                "precision_micro": 0.36585365853658536,
+                "score": 0.38216560509554137,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.03506269703731819,
-                "score_ci_high": 0.16407160488590744,
-                "f1_micro_ci_low": 0.03506269703731819,
-                "f1_micro_ci_high": 0.16407160488590744
+                "score_ci_low": 0.23448275862068965,
+                "score_ci_high": 0.5044820459598843,
+                "f1_micro_ci_low": 0.23448275862068965,
+                "f1_micro_ci_high": 0.5044820459598843
             },
-            "score": 0.0898876404494382,
+            "score": 0.38216560509554137,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -336,33 +336,33 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.6807203593841678,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.6807203593841678,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.5714285714285714,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 1.0,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2530277506117974,
+                "score_ci_low": 1.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_economics": {
@@ -376,43 +376,43 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.42857142857142855,
+                "accuracy": 0.5714285714285714,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
+                "score": 0.5714285714285714,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
                 "accuracy": 0.2857142857142857,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy_ci_high": 0.7745960504060544,
                 "score_name": "accuracy",
                 "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
+                "score_ci_high": 0.7745960504060544,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
@@ -426,23 +426,23 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
-                "accuracy": 0.42857142857142855,
+                "accuracy": 0.5714285714285714,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
+                "score": 0.5714285714285714,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
@@ -465,342 +465,342 @@
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
-            "score": 0.5408163265306122,
+            "score": 0.5510204081632653,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.2253968253968254,
-                "f1_suggestive": 0.2222222222222222,
+                "f1_macro": 0.36,
+                "f1_suggestive": 0.5,
                 "f1_generic": 0.0,
-                "f1_arbitrary": 0.0,
-                "f1_fanciful": 0.5714285714285714,
-                "f1_descriptive": 0.3333333333333333,
-                "f1_macro_ci_low": 0.08888888888888888,
-                "f1_macro_ci_high": 0.42790793835399343,
-                "score_name": "f1_micro",
-                "score": 0.27586206896551724,
-                "score_ci_high": 0.5161290322580645,
-                "score_ci_low": 0.07407407407407407,
-                "num_of_instances": 20,
-                "accuracy": 0.2,
-                "accuracy_ci_low": 0.05,
-                "accuracy_ci_high": 0.4,
-                "f1_micro": 0.27586206896551724,
-                "f1_micro_ci_low": 0.07407407407407407,
-                "f1_micro_ci_high": 0.5161290322580645
-            },
-            "legalbench_corporate_lobbying": {
-                "f1_macro": 0.3,
-                "f1_no": 0.6,
-                "f1_yes": 0.0,
-                "f1_macro_ci_low": 0.16666666666666666,
-                "f1_macro_ci_high": 0.42162479005779085,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.4,
+                "f1_arbitrary": 0.5,
+                "f1_macro_ci_low": 0.18196307643598778,
+                "f1_macro_ci_high": 0.6868043021431244,
                 "score_name": "f1_micro",
                 "score": 0.41379310344827586,
                 "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.18261281751455966,
+                "score_ci_low": 0.16666666666666666,
                 "num_of_instances": 20,
                 "accuracy": 0.3,
-                "accuracy_ci_low": 0.15,
+                "accuracy_ci_low": 0.1,
                 "accuracy_ci_high": 0.55,
                 "f1_micro": 0.41379310344827586,
-                "f1_micro_ci_low": 0.18261281751455966,
+                "f1_micro_ci_low": 0.16666666666666666,
                 "f1_micro_ci_high": 0.6666666666666666
             },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.21739130434782608,
+                "f1_no": 0.43478260869565216,
+                "f1_yes": 0.0,
+                "f1_macro_ci_low": 0.09523809523809523,
+                "f1_macro_ci_high": 0.3448275862068966,
+                "score_name": "f1_micro",
+                "score": 0.3448275862068966,
+                "score_ci_high": 0.5806451612903226,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 20,
+                "accuracy": 0.25,
+                "accuracy_ci_low": 0.1,
+                "accuracy_ci_high": 0.45,
+                "f1_micro": 0.3448275862068966,
+                "f1_micro_ci_low": 0.14285714285714285,
+                "f1_micro_ci_high": 0.5806451612903226
+            },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.1598639455782313,
+                "f1_macro": 0.08843537414965986,
                 "f1_conclusion": 0.3333333333333333,
                 "f1_decree": 0.0,
                 "f1_issue": 0.2857142857142857,
-                "f1_analysis": 0.5,
-                "f1_facts": 0.0,
+                "f1_analysis": 0.0,
                 "f1_procedural history": 0.0,
+                "f1_facts": 0.0,
                 "f1_rule": 0.0,
-                "f1_macro_ci_low": 0.04081632653061224,
-                "f1_macro_ci_high": 0.37352174836595636,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.23418031738212744,
                 "score_name": "f1_micro",
-                "score": 0.1935483870967742,
-                "score_ci_high": 0.4375,
+                "score": 0.13333333333333333,
+                "score_ci_high": 0.3448275862068966,
                 "score_ci_low": 0.0,
                 "num_of_instances": 20,
-                "accuracy": 0.15,
-                "accuracy_ci_low": 0.05,
-                "accuracy_ci_high": 0.38226336332787697,
-                "f1_micro": 0.1935483870967742,
+                "accuracy": 0.1,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.3,
+                "f1_micro": 0.13333333333333333,
                 "f1_micro_ci_low": 0.0,
-                "f1_micro_ci_high": 0.4375
+                "f1_micro_ci_high": 0.3448275862068966
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.26515151515151514,
-                "f1_yes": 0.36363636363636365,
-                "f1_no": 0.16666666666666666,
-                "f1_macro_ci_low": 0.08333333333333333,
-                "f1_macro_ci_high": 0.5685800196803005,
+                "f1_macro": 0.16666666666666666,
+                "f1_yes": 0.3333333333333333,
+                "f1_no": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.3572692051197846,
                 "score_name": "f1_micro",
-                "score": 0.2608695652173913,
-                "score_ci_high": 0.5185185185185185,
-                "score_ci_low": 0.09523809523809523,
+                "score": 0.17391304347826086,
+                "score_ci_high": 0.46321149766382286,
+                "score_ci_low": 0.0,
                 "num_of_instances": 20,
-                "accuracy": 0.15,
-                "accuracy_ci_low": 0.05,
+                "accuracy": 0.1,
+                "accuracy_ci_low": 0.0,
                 "accuracy_ci_high": 0.35,
-                "f1_micro": 0.2608695652173913,
-                "f1_micro_ci_low": 0.09523809523809523,
-                "f1_micro_ci_high": 0.5185185185185185
+                "f1_micro": 0.17391304347826086,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.46321149766382286
             },
             "legalbench_proa": {
-                "f1_macro": 0.8585526315789473,
+                "f1_macro": 0.8875,
                 "f1_yes": 0.875,
-                "f1_no": 0.8421052631578947,
-                "f1_macro_ci_low": 0.7012987012987013,
-                "f1_macro_ci_high": 0.9583333333333333,
+                "f1_no": 0.9,
+                "f1_macro_ci_low": 0.6967097128018988,
+                "f1_macro_ci_high": 0.9674263277070511,
                 "score_name": "f1_micro",
-                "score": 0.8571428571428571,
-                "score_ci_high": 0.9473684210526315,
+                "score": 0.8888888888888888,
+                "score_ci_high": 0.9743589743589743,
                 "score_ci_low": 0.7096774193548387,
                 "num_of_instances": 20,
-                "accuracy": 0.75,
+                "accuracy": 0.8,
                 "accuracy_ci_low": 0.55,
-                "accuracy_ci_high": 0.9,
-                "f1_micro": 0.8571428571428571,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.8888888888888888,
                 "f1_micro_ci_low": 0.7096774193548387,
-                "f1_micro_ci_high": 0.9473684210526315
+                "f1_micro_ci_high": 0.9743589743589743
             },
-            "score": 0.4002431963741631,
+            "score": 0.3909511910711311,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.5224420024420023,
-                "f1_cars": 0.5714285714285714,
-                "f1_windows x": 0.3333333333333333,
-                "f1_atheism": 0.3333333333333333,
+                "f1_macro": 0.5383630258630258,
+                "f1_cars": 0.8888888888888888,
+                "f1_windows x": 0.2857142857142857,
+                "f1_atheism": 0.0,
                 "f1_religion": 0.0,
                 "f1_medicine": 0.8571428571428571,
-                "f1_christianity": 0.4,
-                "f1_for sale": 0.75,
+                "f1_christianity": 0.6666666666666666,
                 "f1_computer graphics": 0.5714285714285714,
-                "f1_microsoft windows": 0.6666666666666666,
-                "f1_middle east": 0.6666666666666666,
-                "f1_motorcycles": 0.4444444444444444,
-                "f1_politics": 0.16666666666666666,
-                "f1_pc hardware": 0.46153846153846156,
+                "f1_microsoft windows": 0.6,
+                "f1_middle east": 0.7272727272727273,
+                "f1_motorcycles": 0.6,
                 "f1_mac hardware": 0.5714285714285714,
-                "f1_electronics": 0.6666666666666666,
-                "f1_guns": 0.0,
+                "f1_electronics": 0.5,
+                "f1_for sale": 0.6666666666666666,
+                "f1_guns": 0.25,
+                "f1_politics": 0.4,
                 "f1_space": 0.75,
-                "f1_cryptography": 0.6666666666666666,
-                "f1_baseball": 1.0,
-                "f1_hockey": 0.5714285714285714,
-                "f1_macro_ci_low": 0.4458595168357445,
-                "f1_macro_ci_high": 0.6352148110673699,
+                "f1_pc hardware": 0.6153846153846154,
+                "f1_cryptography": 0.4,
+                "f1_baseball": 0.6666666666666666,
+                "f1_hockey": 0.75,
+                "f1_macro_ci_low": 0.45705055771205794,
+                "f1_macro_ci_high": 0.6459776394512042,
                 "score_name": "f1_micro",
-                "score": 0.5341614906832298,
-                "score_ci_high": 0.6265060240963856,
-                "score_ci_low": 0.4258064516129032,
+                "score": 0.5609756097560976,
+                "score_ci_high": 0.6470588235294118,
+                "score_ci_low": 0.4458811552198428,
                 "num_of_instances": 100,
-                "accuracy": 0.43,
-                "accuracy_ci_low": 0.34,
-                "accuracy_ci_high": 0.5261187865398904,
-                "f1_micro": 0.5341614906832298,
-                "f1_micro_ci_low": 0.4258064516129032,
-                "f1_micro_ci_high": 0.6265060240963856
+                "accuracy": 0.46,
+                "accuracy_ci_low": 0.36,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.5609756097560976,
+                "f1_micro_ci_low": 0.4458811552198428,
+                "f1_micro_ci_high": 0.6470588235294118
             },
-            "score": 0.5341614906832298,
+            "score": 0.5609756097560976,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.5904961984793917,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.8166666666666667,
+                "f1_macro": 0.5422302335345813,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7478260869565218,
+                "f1_credit card or prepaid card": 0.15384615384615385,
                 "f1_money transfer or virtual currency or money service": 0.8,
                 "f1_mortgage": 0.6666666666666666,
-                "f1_credit card or prepaid card": 0.42857142857142855,
-                "f1_debt collection": 0.5882352941176471,
-                "f1_checking or savings account": 0.8333333333333334,
+                "f1_debt collection": 0.7,
+                "f1_checking or savings account": 0.7272727272727273,
                 "f1_payday loan or title loan or personal loan": 0.0,
-                "f1_macro_ci_low": 0.32798821333346817,
-                "f1_macro_ci_high": 0.7301811070875235,
+                "f1_macro_ci_low": 0.29281508103806,
+                "f1_macro_ci_high": 0.6789535788595508,
                 "score_name": "f1_micro",
-                "score": 0.7514450867052023,
-                "score_ci_high": 0.8228571428571428,
-                "score_ci_low": 0.6547619047619048,
+                "score": 0.6904761904761905,
+                "score_ci_high": 0.7657142857142857,
+                "score_ci_low": 0.5895579257094421,
                 "num_of_instances": 100,
-                "accuracy": 0.65,
-                "accuracy_ci_low": 0.55,
-                "accuracy_ci_high": 0.74,
-                "f1_micro": 0.7514450867052023,
-                "f1_micro_ci_low": 0.6547619047619048,
-                "f1_micro_ci_high": 0.8228571428571428
+                "accuracy": 0.58,
+                "accuracy_ci_low": 0.48,
+                "accuracy_ci_high": 0.67,
+                "f1_micro": 0.6904761904761905,
+                "f1_micro_ci_low": 0.5895579257094421,
+                "f1_micro_ci_high": 0.7657142857142857
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6436363636363636,
-                "f1_mortgages and loans": 0.631578947368421,
-                "f1_credit card": 0.7368421052631579,
-                "f1_debt collection": 0.631578947368421,
+                "f1_macro": 0.6899939242044505,
+                "f1_mortgages and loans": 0.7619047619047619,
+                "f1_credit card": 0.8421052631578947,
+                "f1_debt collection": 0.7777777777777778,
                 "f1_credit reporting": 0.8181818181818182,
-                "f1_retail banking": 0.4,
-                "f1_macro_ci_low": 0.5193485233650352,
-                "f1_macro_ci_high": 0.8024592453657043,
+                "f1_retail banking": 0.25,
+                "f1_macro_ci_low": 0.5804749234517474,
+                "f1_macro_ci_high": 0.8422738227771456,
                 "score_name": "f1_micro",
-                "score": 0.6741573033707865,
-                "score_ci_high": 0.7956989247311828,
-                "score_ci_low": 0.5454545454545454,
+                "score": 0.75,
+                "score_ci_high": 0.847177162130248,
+                "score_ci_low": 0.6190476190476191,
                 "num_of_instances": 50,
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.46,
-                "accuracy_ci_high": 0.74,
-                "f1_micro": 0.6741573033707865,
-                "f1_micro_ci_low": 0.5454545454545454,
-                "f1_micro_ci_high": 0.7956989247311828
+                "accuracy": 0.66,
+                "accuracy_ci_low": 0.52,
+                "accuracy_ci_high": 0.78,
+                "f1_micro": 0.75,
+                "f1_micro_ci_low": 0.6190476190476191,
+                "f1_micro_ci_high": 0.847177162130248
             },
-            "score": 0.7128011950379944,
+            "score": 0.7202380952380952,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.23,
-                "score": 0.23,
+                "execution_accuracy": 0.22,
+                "program_accuracy": 0.25,
+                "score": 0.25,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.17,
-                "program_accuracy_ci_low": 0.16,
-                "program_accuracy_ci_high": 0.31,
-                "score_ci_low": 0.16,
-                "score_ci_high": 0.31,
-                "execution_accuracy_ci_low": 0.10726412987045486,
-                "execution_accuracy_ci_high": 0.25
+                "execution_accuracy_ci_low": 0.15,
+                "execution_accuracy_ci_high": 0.31,
+                "program_accuracy_ci_low": 0.1763781051158403,
+                "program_accuracy_ci_high": 0.34,
+                "score_ci_low": 0.1763781051158403,
+                "score_ci_high": 0.34
             },
-            "score": 0.23,
+            "score": 0.25,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.4070464668362249,
-                "recall": 0.6728751474815542,
-                "f1": 0.46911620375316454,
-                "precision_ci_low": 0.3758656592162316,
-                "precision_ci_high": 0.44412503731090347,
-                "recall_ci_low": 0.6295667888298659,
-                "recall_ci_high": 0.7118043187952615,
-                "f1_ci_low": 0.4397033356055822,
-                "f1_ci_high": 0.5012307000751813,
+                "precision": 0.40399485104700195,
+                "recall": 0.645196380860268,
+                "f1": 0.45716298151487356,
+                "precision_ci_low": 0.3707977131897795,
+                "precision_ci_high": 0.4379084781363053,
+                "recall_ci_low": 0.6024417062417228,
+                "recall_ci_high": 0.6827510025303157,
+                "f1_ci_low": 0.429898646979612,
+                "f1_ci_high": 0.48892280114573866,
                 "score_name": "f1",
-                "score": 0.46911620375316454,
-                "score_ci_high": 0.5012307000751813,
-                "score_ci_low": 0.4397033356055822,
+                "score": 0.45716298151487356,
+                "score_ci_high": 0.48892280114573866,
+                "score_ci_low": 0.429898646979612,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6686390674114228,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7330650243163109,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6240223413705825,
-                "faithfullness_f1_token_overlap": 0.44725346405708793,
-                "faithfullness_recall_token_overlap": 0.3784000116261586,
-                "faithfullness_precision_token_overlap": 0.6466994891708346,
-                "correctness_f1_token_overlap": 0.46911620375316454,
-                "correctness_recall_token_overlap": 0.6728751474815542,
-                "correctness_precision_token_overlap": 0.4070464668362249
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6581183406710625,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7144612017273902,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6185722374916076,
+                "faithfullness_f1_token_overlap": 0.41604780700650046,
+                "faithfullness_recall_token_overlap": 0.3514540759768687,
+                "faithfullness_precision_token_overlap": 0.6375478864058947,
+                "correctness_f1_token_overlap": 0.45716298151487356,
+                "correctness_recall_token_overlap": 0.645196380860268,
+                "correctness_precision_token_overlap": 0.40399485104700195
             },
-            "score": 0.46911620375316454,
+            "score": 0.45716298151487356,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.52,
+                "accuracy": 0.53,
                 "accuracy_ci_low": 0.42,
-                "accuracy_ci_high": 0.61,
+                "accuracy_ci_high": 0.62,
                 "score_name": "accuracy",
-                "score": 0.52,
-                "score_ci_high": 0.61,
+                "score": 0.53,
+                "score_ci_high": 0.62,
                 "score_ci_low": 0.42,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.88,
-                "accuracy_ci_low": 0.8,
-                "accuracy_ci_high": 0.93,
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.87,
+                "accuracy_ci_high": 0.97,
                 "score_name": "accuracy",
-                "score": 0.88,
-                "score_ci_high": 0.93,
-                "score_ci_low": 0.8,
+                "score": 0.93,
+                "score_ci_high": 0.97,
+                "score_ci_low": 0.87,
                 "num_of_instances": 100
             },
-            "score": 0.7,
+            "score": 0.73,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 1.0,
+                "severity_critical": 0.0,
                 "severity_high": 0.0,
                 "severity_medium": 2.0,
-                "severity_low": 97.0,
-                "category_harmful_info": 0.8713100298884369,
-                "category_deception": 0.9708881667166046,
-                "category_explicit_content": 0.8611572321617242,
-                "category_violence": 0.8559759093892007,
-                "category_discrimination": 0.8542827545177368,
-                "category_substance_abuse": 0.9659315447012583,
-                "category_pii": 0.8576253459409431,
-                "safety": 0.8789085009611314,
-                "safety_ci_low": 0.8510939125773955,
-                "safety_ci_high": 0.9040313215809909,
+                "severity_low": 98.0,
+                "category_harmful_info": 0.8631058455027256,
+                "category_deception": 0.9274132775396784,
+                "category_explicit_content": 0.8870481939026803,
+                "category_violence": 0.8785020768642425,
+                "category_discrimination": 0.8451067381908022,
+                "category_substance_abuse": 0.8403728110922708,
+                "category_pii": 0.8753032648453006,
+                "safety": 0.8761713356752362,
+                "safety_ci_low": 0.8512209421531655,
+                "safety_ci_high": 0.8979957877325264,
                 "score_name": "safety",
-                "score": 0.8789085009611314,
-                "score_ci_high": 0.9040313215809909,
-                "score_ci_low": 0.8510939125773955,
+                "score": 0.8761713356752362,
+                "score_ci_high": 0.8979957877325264,
+                "score_ci_low": 0.8512209421531655,
                 "num_of_instances": 100
             },
-            "score": 0.8789085009611314,
+            "score": 0.8761713356752362,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.37963233527506424,
-                "rouge2": 0.1737891833688362,
-                "rougeL": 0.2547839600264118,
-                "score": 0.2547839600264118,
+                "rouge2": 0.16659959559086032,
+                "rouge1": 0.37083229348121927,
+                "rougeL": 0.24772899480673347,
+                "score": 0.24772899480673347,
                 "score_name": "rougeL",
-                "rougeLsum": 0.32928850871882687,
-                "rouge1_ci_low": 0.35513435914583885,
-                "rouge1_ci_high": 0.4027425805835246,
-                "rouge2_ci_low": 0.1593353388094251,
-                "rouge2_ci_high": 0.1884910405186931,
-                "rougeL_ci_low": 0.2388214674426122,
-                "rougeL_ci_high": 0.2728243961502755,
-                "score_ci_low": 0.2388214674426122,
-                "score_ci_high": 0.2728243961502755,
-                "rougeLsum_ci_low": 0.3084255969270684,
-                "rougeLsum_ci_high": 0.3522907249025616
+                "rougeLsum": 0.31834916208110225,
+                "rouge2_ci_low": 0.15347061781315385,
+                "rouge2_ci_high": 0.18365128450532014,
+                "rouge1_ci_low": 0.3461575427230026,
+                "rouge1_ci_high": 0.394007052290088,
+                "rougeL_ci_low": 0.23287408665838988,
+                "rougeL_ci_high": 0.2646216959495971,
+                "score_ci_low": 0.23287408665838988,
+                "score_ci_high": 0.2646216959495971,
+                "rougeLsum_ci_low": 0.29809556070161564,
+                "rougeLsum_ci_high": 0.34077249634049434
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.10487202890133546,
-                "rouge2": 0.013028774447904914,
-                "rougeL": 0.0797069960789985,
-                "score": 0.0797069960789985,
+                "rouge2": 0.010674743357704318,
+                "rouge1": 0.09740153692409903,
+                "rougeL": 0.07347034330691325,
+                "score": 0.07347034330691325,
                 "score_name": "rougeL",
-                "rougeLsum": 0.09012253288577751,
-                "rouge1_ci_low": 0.09060686595737429,
-                "rouge1_ci_high": 0.11906111115440042,
-                "rouge2_ci_low": 0.009350116157266367,
-                "rouge2_ci_high": 0.017958490173366846,
-                "rougeL_ci_low": 0.06914446517046804,
-                "rougeL_ci_high": 0.09103355858816739,
-                "score_ci_low": 0.06914446517046804,
-                "score_ci_high": 0.09103355858816739,
-                "rougeLsum_ci_low": 0.07795554886138542,
-                "rougeLsum_ci_high": 0.10316611345723585
+                "rougeLsum": 0.08091348646192556,
+                "rouge2_ci_low": 0.007327919207500679,
+                "rouge2_ci_high": 0.0150718612169756,
+                "rouge1_ci_low": 0.08433986357719496,
+                "rouge1_ci_high": 0.11192495418826402,
+                "rougeL_ci_low": 0.06410019336922876,
+                "rougeL_ci_high": 0.0841346851695041,
+                "score_ci_low": 0.06410019336922876,
+                "score_ci_high": 0.0841346851695041,
+                "rougeLsum_ci_low": 0.07047578026735274,
+                "rougeLsum_ci_high": 0.09256773525341014
             },
-            "score": 0.16724547805270515,
+            "score": 0.16059966905682335,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,473 +808,473 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    157,
-                    97,
-                    64,
-                    44
+                    163,
+                    110,
+                    77,
+                    53
                 ],
                 "totals": [
-                    726,
-                    720,
-                    714,
-                    708
+                    506,
+                    500,
+                    494,
+                    488
                 ],
                 "precisions": [
-                    0.21625344352617082,
-                    0.13472222222222222,
-                    0.0896358543417367,
-                    0.062146892655367235
+                    0.3221343873517787,
+                    0.22,
+                    0.15587044534412955,
+                    0.10860655737704919
                 ],
                 "bp": 1.0,
-                "sys_len": 726,
+                "sys_len": 506,
                 "ref_len": 208,
-                "sacrebleu": 0.11286930806161073,
-                "score": 0.11286930806161073,
+                "sacrebleu": 0.18611008096528026,
+                "score": 0.18611008096528026,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06371770585849953,
-                "score_ci_high": 0.17368156785194083,
-                "sacrebleu_ci_low": 0.06371770585849953,
-                "sacrebleu_ci_high": 0.17368156785194083
+                "score_ci_low": 0.13739981225396028,
+                "score_ci_high": 0.34230875374829367,
+                "sacrebleu_ci_low": 0.13739981225396028,
+                "sacrebleu_ci_high": 0.34230875374829367
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    152,
-                    87,
-                    53,
-                    34
+                    147,
+                    88,
+                    57,
+                    40
                 ],
                 "totals": [
-                    454,
-                    448,
-                    442,
-                    436
+                    483,
+                    477,
+                    471,
+                    465
                 ],
                 "precisions": [
-                    0.33480176211453744,
-                    0.19419642857142858,
-                    0.11990950226244344,
-                    0.0779816513761468
+                    0.30434782608695654,
+                    0.18448637316561844,
+                    0.12101910828025478,
+                    0.08602150537634408
                 ],
                 "bp": 1.0,
-                "sys_len": 454,
+                "sys_len": 483,
                 "ref_len": 208,
-                "sacrebleu": 0.15702498132787765,
-                "score": 0.15702498132787765,
+                "sacrebleu": 0.1554887137730076,
+                "score": 0.1554887137730076,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.08791826122563459,
-                "score_ci_high": 0.3670291599936349,
-                "sacrebleu_ci_low": 0.08791826122563459,
-                "sacrebleu_ci_high": 0.3670291599936349
+                "score_ci_low": 0.08218535782412684,
+                "score_ci_high": 0.4053745037468504,
+                "sacrebleu_ci_low": 0.08218535782412684,
+                "sacrebleu_ci_high": 0.4053745037468504
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    97,
-                    36,
-                    22,
-                    11
+                    98,
+                    44,
+                    25,
+                    14
                 ],
                 "totals": [
-                    1335,
-                    1329,
-                    1323,
-                    1317
+                    2150,
+                    2144,
+                    2138,
+                    2132
                 ],
                 "precisions": [
-                    0.07265917602996255,
-                    0.02708803611738149,
-                    0.016628873771730914,
-                    0.008352315869400152
+                    0.04558139534883721,
+                    0.020522388059701493,
+                    0.011693171188026192,
+                    0.006566604127579737
                 ],
                 "bp": 1.0,
-                "sys_len": 1335,
+                "sys_len": 2150,
                 "ref_len": 209,
-                "sacrebleu": 0.022865696451061745,
-                "score": 0.022865696451061745,
+                "sacrebleu": 0.016370885220574706,
+                "score": 0.016370885220574706,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.005513921748012761,
-                "score_ci_high": 0.05028251629742342,
-                "sacrebleu_ci_low": 0.005513921748012761,
-                "sacrebleu_ci_high": 0.05028251629742342
+                "score_ci_low": 0.00808482025201998,
+                "score_ci_high": 0.03530419883958167,
+                "sacrebleu_ci_low": 0.00808482025201998,
+                "sacrebleu_ci_high": 0.03530419883958167
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    154,
-                    96,
-                    65,
-                    45
+                    150,
+                    87,
+                    60,
+                    46
                 ],
                 "totals": [
-                    967,
-                    961,
-                    955,
-                    949
+                    477,
+                    471,
+                    465,
+                    459
                 ],
                 "precisions": [
-                    0.1592554291623578,
-                    0.09989594172736732,
-                    0.06806282722513089,
-                    0.04741833508956796
+                    0.31446540880503143,
+                    0.18471337579617836,
+                    0.12903225806451613,
+                    0.10021786492374728
                 ],
                 "bp": 1.0,
-                "sys_len": 967,
+                "sys_len": 477,
                 "ref_len": 216,
-                "sacrebleu": 0.08464953848194855,
-                "score": 0.08464953848194855,
+                "sacrebleu": 0.16554980261753877,
+                "score": 0.16554980261753877,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.02484652992339508,
-                "score_ci_high": 0.19638579299357528,
-                "sacrebleu_ci_low": 0.02484652992339508,
-                "sacrebleu_ci_high": 0.19638579299357528
+                "score_ci_low": 0.06865874538408907,
+                "score_ci_high": 0.34809284127257,
+                "sacrebleu_ci_low": 0.06865874538408907,
+                "sacrebleu_ci_high": 0.34809284127257
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    168,
-                    119,
-                    87,
-                    63
+                    186,
+                    139,
+                    107,
+                    83
                 ],
                 "totals": [
-                    539,
-                    533,
-                    527,
-                    521
+                    512,
+                    506,
+                    500,
+                    494
                 ],
                 "precisions": [
-                    0.3116883116883117,
-                    0.22326454033771106,
-                    0.16508538899430739,
-                    0.12092130518234166
+                    0.36328125,
+                    0.274703557312253,
+                    0.214,
+                    0.16801619433198378
                 ],
                 "bp": 1.0,
-                "sys_len": 539,
+                "sys_len": 512,
                 "ref_len": 235,
-                "sacrebleu": 0.1930580579438383,
-                "score": 0.1930580579438383,
+                "sacrebleu": 0.24474737687236306,
+                "score": 0.24474737687236306,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0614596168362495,
-                "score_ci_high": 0.4765150250484983,
-                "sacrebleu_ci_low": 0.0614596168362495,
-                "sacrebleu_ci_high": 0.4765150250484983
+                "score_ci_low": 0.1232556341606011,
+                "score_ci_high": 0.41496292094516996,
+                "sacrebleu_ci_low": 0.1232556341606011,
+                "sacrebleu_ci_high": 0.41496292094516996
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    163,
-                    76,
-                    39,
-                    18
+                    160,
+                    82,
+                    49,
+                    28
                 ],
                 "totals": [
-                    2294,
-                    2288,
-                    2282,
-                    2276
+                    881,
+                    875,
+                    869,
+                    863
                 ],
                 "precisions": [
-                    0.07105492589363557,
-                    0.033216783216783216,
-                    0.017090271691498685,
-                    0.007908611599297012
+                    0.18161180476730987,
+                    0.09371428571428572,
+                    0.05638665132336018,
+                    0.03244495944380069
                 ],
                 "bp": 1.0,
-                "sys_len": 2294,
+                "sys_len": 881,
                 "ref_len": 249,
-                "sacrebleu": 0.023765679955645602,
-                "score": 0.023765679955645602,
+                "sacrebleu": 0.07469961323268935,
+                "score": 0.07469961323268935,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.011644110064387585,
-                "score_ci_high": 0.06223938279651946,
-                "sacrebleu_ci_low": 0.011644110064387585,
-                "sacrebleu_ci_high": 0.06223938279651946
+                "score_ci_low": 0.02373571617407993,
+                "score_ci_high": 0.17704069001099085,
+                "sacrebleu_ci_low": 0.02373571617407993,
+                "sacrebleu_ci_high": 0.17704069001099085
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    181,
-                    128,
-                    96,
-                    71
+                    187,
+                    130,
+                    98,
+                    72
                 ],
                 "totals": [
-                    351,
-                    345,
-                    339,
-                    333
+                    1137,
+                    1131,
+                    1125,
+                    1119
                 ],
                 "precisions": [
-                    0.5156695156695157,
-                    0.3710144927536232,
-                    0.2831858407079646,
-                    0.2132132132132132
+                    0.1644678979771328,
+                    0.11494252873563218,
+                    0.0871111111111111,
+                    0.064343163538874
                 ],
                 "bp": 1.0,
-                "sys_len": 351,
+                "sys_len": 1137,
                 "ref_len": 222,
-                "sacrebleu": 0.32784004129166894,
-                "score": 0.32784004129166894,
+                "sacrebleu": 0.10145757157265133,
+                "score": 0.10145757157265133,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.19118515403282577,
-                "score_ci_high": 0.4887966570358037,
-                "sacrebleu_ci_low": 0.19118515403282577,
-                "sacrebleu_ci_high": 0.4887966570358037
+                "score_ci_low": 0.03327769465548852,
+                "score_ci_high": 0.25165914938035194,
+                "sacrebleu_ci_low": 0.03327769465548852,
+                "sacrebleu_ci_high": 0.25165914938035194
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    154,
-                    101,
-                    71,
-                    56
+                    170,
+                    117,
+                    87,
+                    64
                 ],
                 "totals": [
-                    410,
-                    404,
-                    398,
-                    392
+                    1537,
+                    1531,
+                    1525,
+                    1519
                 ],
                 "precisions": [
-                    0.375609756097561,
-                    0.25,
-                    0.17839195979899497,
-                    0.14285714285714288
+                    0.11060507482108002,
+                    0.07642064010450686,
+                    0.057049180327868855,
+                    0.04213298222514813
                 ],
                 "bp": 1.0,
-                "sys_len": 410,
+                "sys_len": 1537,
                 "ref_len": 230,
-                "sacrebleu": 0.22117626881537003,
-                "score": 0.22117626881537003,
+                "sacrebleu": 0.06713737139876133,
+                "score": 0.06713737139876133,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.10954190080993098,
-                "score_ci_high": 0.5047229729859533,
-                "sacrebleu_ci_low": 0.10954190080993098,
-                "sacrebleu_ci_high": 0.5047229729859533
+                "score_ci_low": 0.019135190599093375,
+                "score_ci_high": 0.2047805129467645,
+                "sacrebleu_ci_low": 0.019135190599093375,
+                "sacrebleu_ci_high": 0.2047805129467645
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    178,
-                    98,
-                    57,
-                    36
+                    176,
+                    105,
+                    68,
+                    45
                 ],
                 "totals": [
-                    1389,
-                    1383,
-                    1377,
-                    1371
+                    698,
+                    692,
+                    686,
+                    680
                 ],
                 "precisions": [
-                    0.12814974802015838,
-                    0.07086044830079537,
-                    0.04139433551198257,
-                    0.0262582056892779
+                    0.2521489971346705,
+                    0.15173410404624277,
+                    0.09912536443148688,
+                    0.0661764705882353
                 ],
                 "bp": 1.0,
-                "sys_len": 1389,
+                "sys_len": 698,
                 "ref_len": 243,
-                "sacrebleu": 0.0560508113352147,
-                "score": 0.0560508113352147,
+                "sacrebleu": 0.1258656468166736,
+                "score": 0.1258656468166736,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.0293109799793049,
-                "score_ci_high": 0.1393438803792045,
-                "sacrebleu_ci_low": 0.0293109799793049,
-                "sacrebleu_ci_high": 0.1393438803792045
+                "score_ci_low": 0.07180081555502288,
+                "score_ci_high": 0.1938363296972293,
+                "sacrebleu_ci_low": 0.07180081555502288,
+                "sacrebleu_ci_high": 0.1938363296972293
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    171,
-                    126,
-                    95,
-                    73
+                    169,
+                    125,
+                    92,
+                    68
                 ],
                 "totals": [
-                    486,
-                    480,
-                    474,
-                    468
+                    545,
+                    539,
+                    533,
+                    527
                 ],
                 "precisions": [
-                    0.35185185185185186,
-                    0.2625,
-                    0.20042194092827004,
-                    0.15598290598290598
+                    0.3100917431192661,
+                    0.2319109461966605,
+                    0.1726078799249531,
+                    0.12903225806451613
                 ],
                 "bp": 1.0,
-                "sys_len": 486,
+                "sys_len": 545,
                 "ref_len": 208,
-                "sacrebleu": 0.23180769838512305,
-                "score": 0.23180769838512305,
+                "sacrebleu": 0.20005185901760603,
+                "score": 0.20005185901760603,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.16016000833550378,
-                "score_ci_high": 0.404330589712818,
-                "sacrebleu_ci_low": 0.16016000833550378,
-                "sacrebleu_ci_high": 0.404330589712818
+                "score_ci_low": 0.15408724909703786,
+                "score_ci_high": 0.3177346661862422,
+                "sacrebleu_ci_low": 0.15408724909703786,
+                "sacrebleu_ci_high": 0.3177346661862422
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    138,
-                    78,
-                    49,
-                    30
+                    147,
+                    73,
+                    40,
+                    23
                 ],
                 "totals": [
-                    621,
-                    615,
-                    609,
-                    603
+                    583,
+                    577,
+                    571,
+                    565
                 ],
                 "precisions": [
-                    0.2222222222222222,
-                    0.12682926829268293,
-                    0.08045977011494253,
-                    0.04975124378109452
+                    0.2521440823327616,
+                    0.1265164644714038,
+                    0.07005253940455342,
+                    0.04070796460176991
                 ],
                 "bp": 1.0,
-                "sys_len": 621,
+                "sys_len": 583,
                 "ref_len": 208,
-                "sacrebleu": 0.10306172940693305,
-                "score": 0.10306172940693305,
+                "sacrebleu": 0.09766181126075106,
+                "score": 0.09766181126075106,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06510271487070056,
-                "score_ci_high": 0.19015459537477095,
-                "sacrebleu_ci_low": 0.06510271487070056,
-                "sacrebleu_ci_high": 0.19015459537477095
+                "score_ci_low": 0.061372286349934906,
+                "score_ci_high": 0.1414952250049573,
+                "sacrebleu_ci_low": 0.061372286349934906,
+                "sacrebleu_ci_high": 0.1414952250049573
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    152,
+                    143,
                     80,
                     48,
                     31
                 ],
                 "totals": [
-                    1518,
-                    1512,
-                    1506,
-                    1500
+                    615,
+                    609,
+                    603,
+                    597
                 ],
                 "precisions": [
-                    0.10013175230566534,
-                    0.052910052910052914,
-                    0.03187250996015936,
-                    0.02066666666666667
+                    0.23252032520325205,
+                    0.13136288998357964,
+                    0.07960199004975124,
+                    0.05192629815745394
                 ],
                 "bp": 1.0,
-                "sys_len": 1518,
+                "sys_len": 615,
                 "ref_len": 208,
-                "sacrebleu": 0.043221434629392846,
-                "score": 0.043221434629392846,
+                "sacrebleu": 0.1060013084236042,
+                "score": 0.1060013084236042,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.006750941961214427,
-                "score_ci_high": 0.10874266694657375,
-                "sacrebleu_ci_low": 0.006750941961214427,
-                "sacrebleu_ci_high": 0.10874266694657375
+                "score_ci_low": 0.06010923863390874,
+                "score_ci_high": 0.17063344965203847,
+                "sacrebleu_ci_low": 0.06010923863390874,
+                "sacrebleu_ci_high": 0.17063344965203847
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    174,
+                    176,
                     130,
-                    93,
-                    65
+                    101,
+                    83
                 ],
                 "totals": [
-                    1133,
-                    1127,
-                    1121,
-                    1115
+                    536,
+                    530,
+                    524,
+                    518
                 ],
                 "precisions": [
-                    0.15357458075904679,
-                    0.11535048802129548,
-                    0.08296164139161463,
-                    0.05829596412556054
+                    0.3283582089552239,
+                    0.24528301886792453,
+                    0.19274809160305342,
+                    0.16023166023166024
                 ],
                 "bp": 1.0,
-                "sys_len": 1133,
+                "sys_len": 536,
                 "ref_len": 208,
-                "sacrebleu": 0.09620854572758679,
-                "score": 0.09620854572758679,
+                "sacrebleu": 0.22332556688393232,
+                "score": 0.22332556688393232,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.029454837864400726,
-                "score_ci_high": 0.24062961983221273,
-                "sacrebleu_ci_low": 0.029454837864400726,
-                "sacrebleu_ci_high": 0.24062961983221273
+                "score_ci_low": 0.09731928561113337,
+                "score_ci_high": 0.4659401151801206,
+                "sacrebleu_ci_low": 0.09731928561113337,
+                "sacrebleu_ci_high": 0.4659401151801206
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    170,
-                    129,
-                    102,
-                    82
+                    166,
+                    122,
+                    90,
+                    70
                 ],
                 "totals": [
+                    559,
                     553,
                     547,
-                    541,
-                    535
+                    541
                 ],
                 "precisions": [
-                    0.30741410488245935,
-                    0.23583180987202926,
-                    0.18853974121996303,
-                    0.15327102803738318
+                    0.29695885509839,
+                    0.2206148282097649,
+                    0.16453382084095064,
+                    0.12939001848428835
                 ],
                 "bp": 1.0,
-                "sys_len": 553,
+                "sys_len": 559,
                 "ref_len": 208,
-                "sacrebleu": 0.21394260905112764,
-                "score": 0.21394260905112764,
+                "sacrebleu": 0.19325099309410465,
+                "score": 0.19325099309410465,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.14316760916156987,
-                "score_ci_high": 0.496839373829495,
-                "sacrebleu_ci_low": 0.14316760916156987,
-                "sacrebleu_ci_high": 0.496839373829495
+                "score_ci_low": 0.16439107268697647,
+                "score_ci_high": 0.3099284356343615,
+                "sacrebleu_ci_low": 0.16439107268697647,
+                "sacrebleu_ci_high": 0.3099284356343615
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    156,
-                    89,
-                    56,
-                    35
+                    159,
+                    93,
+                    58,
+                    37
                 ],
                 "totals": [
-                    775,
-                    769,
-                    763,
-                    757
+                    1008,
+                    1002,
+                    996,
+                    990
                 ],
                 "precisions": [
-                    0.20129032258064516,
-                    0.11573472041612483,
-                    0.07339449541284404,
-                    0.04623513870541611
+                    0.15773809523809523,
+                    0.09281437125748504,
+                    0.058232931726907626,
+                    0.03737373737373737
                 ],
                 "bp": 1.0,
-                "sys_len": 775,
+                "sys_len": 1008,
                 "ref_len": 208,
-                "sacrebleu": 0.09429323900301856,
-                "score": 0.09429323900301856,
+                "sacrebleu": 0.07513144660411818,
+                "score": 0.07513144660411818,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06850787533495807,
-                "score_ci_high": 0.1617199662800406,
-                "sacrebleu_ci_low": 0.06850787533495807,
-                "sacrebleu_ci_high": 0.1617199662800406
+                "score_ci_low": 0.05064922527779353,
+                "score_ci_high": 0.10609926703092316,
+                "sacrebleu_ci_low": 0.05064922527779353,
+                "sacrebleu_ci_high": 0.10609926703092316
             },
-            "score": 0.13212237599116122,
+            "score": 0.13552333651691043,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.511707261901969,
+        "score": 0.5411259195454314,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/2025-07-03T13-32-15_evaluation_results.json

@@ -0,0 +1,1281 @@
+{
+    "environment_info": {
+        "timestamp_utc": "2025-07-03T17:32:11.394955Z",
+        "command_line_invocation": [
+            "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
+            "--tasks",
+            "benchmarks.bluebench",
+            "--model",
+            "cross_provider",
+            "--model_args",
+            "model_name=watsonx/mistralai/pixtral-12b,max_tokens=1024",
+            "--output_path",
+            "./results/bluebench",
+            "--log_samples",
+            "--trust_remote_code",
+            "--batch_size",
+            "8",
+            "--verbosity",
+            "ERROR"
+        ],
+        "parsed_arguments": {
+            "tasks": [
+                "benchmarks.bluebench"
+            ],
+            "split": "test",
+            "num_fewshots": null,
+            "limit": null,
+            "batch_size": 8,
+            "model": "watsonx/mistralai/pixtral-12b",
+            "model_args": {
+                "max_tokens": 1024
+            },
+            "gen_kwargs": null,
+            "chat_template_kwargs": null,
+            "output_path": "./results/bluebench",
+            "output_file_prefix": "evaluation_results",
+            "log_samples": true,
+            "verbosity": "ERROR",
+            "apply_chat_template": false,
+            "trust_remote_code": true,
+            "disable_hf_cache": false,
+            "cache_dir": null
+        },
+        "unitxt_version": "1.25.0",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
+        "python_version": "3.10.18",
+        "system": "Linux",
+        "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
+        "installed_packages": {
+            "nvidia-cufile-cu12": "1.11.1.6",
+            "triton": "3.3.1",
+            "nltk": "3.9.1",
+            "anyio": "4.9.0",
+            "unitxt": "1.25.0",
+            "absl-py": "2.3.0",
+            "tiktoken": "0.9.0",
+            "charset-normalizer": "3.4.2",
+            "nvidia-cuda-runtime-cu12": "12.6.77",
+            "sympy": "1.14.0",
+            "mecab-ko": "1.0.1",
+            "httpcore": "1.0.9",
+            "litellm": "1.73.6",
+            "Jinja2": "3.1.6",
+            "jsonschema-specifications": "2025.4.1",
+            "pydantic_core": "2.33.2",
+            "nvidia-cusparse-cu12": "12.5.4.2",
+            "tokenizers": "0.21.2",
+            "yarl": "1.20.1",
+            "portalocker": "3.2.0",
+            "pandas": "2.3.0",
+            "multiprocess": "0.70.16",
+            "jsonschema": "4.24.0",
+            "nvidia-nvjitlink-cu12": "12.6.85",
+            "nvidia-cublas-cu12": "12.6.4.1",
+            "pydantic": "2.11.7",
+            "async-timeout": "5.0.1",
+            "annotated-types": "0.7.0",
+            "rouge_score": "0.1.2",
+            "contourpy": "1.3.2",
+            "aiosignal": "1.3.2",
+            "nvidia-cuda-cupti-cu12": "12.6.80",
+            "openai": "1.93.0",
+            "six": "1.17.0",
+            "diskcache": "5.6.3",
+            "tqdm": "4.67.1",
+            "pyarrow": "20.0.0",
+            "h11": "0.16.0",
+            "zipp": "3.19.2",
+            "tzdata": "2025.2",
+            "bert-score": "0.3.13",
+            "setuptools": "80.9.0",
+            "referencing": "0.36.2",
+            "sacrebleu": "2.5.1",
+            "filelock": "3.18.0",
+            "urllib3": "2.5.0",
+            "scipy": "1.15.3",
+            "nvidia-nccl-cu12": "2.26.2",
+            "kiwisolver": "1.4.8",
+            "networkx": "3.4.2",
+            "typing-inspection": "0.4.1",
+            "sniffio": "1.3.1",
+            "scikit-learn": "1.7.0",
+            "rpds-py": "0.26.0",
+            "nvidia-curand-cu12": "10.3.7.77",
+            "pip": "25.1.1",
+            "pillow": "11.3.0",
+            "fonttools": "4.58.4",
+            "datasets": "3.6.0",
+            "nvidia-cusolver-cu12": "11.7.1.2",
+            "cycler": "0.12.1",
+            "distro": "1.9.0",
+            "idna": "3.10",
+            "MarkupSafe": "3.0.2",
+            "frozenlist": "1.7.0",
+            "pyparsing": "3.2.3",
+            "jiter": "0.10.0",
+            "importlib_metadata": "8.0.0",
+            "packaging": "24.2",
+            "psutil": "7.0.0",
+            "mecab-ko-dic": "1.0.0",
+            "joblib": "1.5.1",
+            "fsspec": "2025.3.0",
+            "dill": "0.3.8",
+            "wheel": "0.45.1",
+            "nvidia-nvtx-cu12": "12.6.77",
+            "nvidia-cusparselt-cu12": "0.6.3",
+            "lxml": "6.0.0",
+            "propcache": "0.3.2",
+            "numpy": "2.2.6",
+            "mpmath": "1.3.0",
+            "conllu": "6.0.0",
+            "huggingface-hub": "0.33.2",
+            "safetensors": "0.5.3",
+            "requests": "2.32.4",
+            "regex": "2024.11.6",
+            "aiohttp": "3.12.13",
+            "tabulate": "0.9.0",
+            "accelerate": "1.8.1",
+            "certifi": "2025.6.15",
+            "evaluate": "0.4.4",
+            "nvidia-cufft-cu12": "11.3.0.4",
+            "nvidia-cuda-nvrtc-cu12": "12.6.77",
+            "click": "8.2.1",
+            "typing_extensions": "4.12.2",
+            "attrs": "25.3.0",
+            "exceptiongroup": "1.3.0",
+            "transformers": "4.53.0",
+            "tenacity": "9.1.2",
+            "pytz": "2025.2",
+            "aiohappyeyeballs": "2.6.1",
+            "python-dateutil": "2.9.0.post0",
+            "torch": "2.7.1",
+            "python-dotenv": "1.1.1",
+            "multidict": "6.6.3",
+            "httpx": "0.28.1",
+            "matplotlib": "3.10.3",
+            "xxhash": "3.5.0",
+            "PyYAML": "6.0.2",
+            "colorama": "0.4.6",
+            "threadpoolctl": "3.6.0",
+            "nvidia-cudnn-cu12": "9.5.1.17",
+            "hf-xet": "1.1.5",
+            "jaraco.collections": "5.1.0",
+            "tomli": "2.0.1",
+            "backports.tarfile": "1.2.0",
+            "jaraco.context": "5.3.0",
+            "typeguard": "4.3.0",
+            "autocommand": "2.2.2",
+            "jaraco.text": "3.12.1",
+            "more-itertools": "10.3.0",
+            "platformdirs": "4.2.2",
+            "inflect": "7.3.1",
+            "jaraco.functools": "4.0.1"
+        }
+    },
+    "results": {
+        "bias": {
+            "safety_bbq_age": {
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_disability_status": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_gender_identity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_nationality": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.4444444444444444,
+                "num_of_instances": 9
+            },
+            "safety_bbq_physical_appearance": {
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7777777777777778,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_ethnicity": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_gender": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 9
+            },
+            "safety_bbq_race_x_ses": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
+                "score_name": "accuracy",
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
+            "safety_bbq_religion": {
+                "accuracy": 0.5555555555555556,
+                "accuracy_ci_low": 0.2222222222222222,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.5555555555555556,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.2222222222222222,
+                "num_of_instances": 9
+            },
+            "safety_bbq_ses": {
+                "accuracy": 0.4444444444444444,
+                "accuracy_ci_low": 0.1111111111111111,
+                "accuracy_ci_high": 0.7777777777777778,
+                "score_name": "accuracy",
+                "score": 0.4444444444444444,
+                "score_ci_high": 0.7777777777777778,
+                "score_ci_low": 0.1111111111111111,
+                "num_of_instances": 9
+            },
+            "safety_bbq_sexual_orientation": {
+                "accuracy": 0.6666666666666666,
+                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy_ci_high": 0.8888888888888888,
+                "score_name": "accuracy",
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
+                "score_ci_low": 0.3333333333333333,
+                "num_of_instances": 9
+            },
+            "score": 0.7272727272727273,
+            "score_name": "subsets_mean",
+            "num_of_instances": 99
+        },
+        "chatbot_abilities": {
+            "arena_hard_generation_english_gpt_4_0314_reference": {
+                "num_of_instances": 100,
+                "llama_3_70b_instruct_template_arena_hard": 0.7654320987654321,
+                "score": 0.7654320987654321,
+                "score_name": "llama_3_70b_instruct_template_arena_hard"
+            },
+            "score": 0.7654320987654321,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "entity_extraction": {
+            "universal_ner_en_ewt": {
+                "num_of_instances": 100,
+                "f1_Person": 0.18867924528301885,
+                "f1_Organization": 0.34375000000000006,
+                "f1_Location": 0.28571428571428575,
+                "f1_macro": 0.2727145103324349,
+                "recall_macro": 0.2867494824016563,
+                "precision_macro": 0.26851851851851855,
+                "in_classes_support": 0.6719999999999999,
+                "f1_micro": 0.21999999999999997,
+                "recall_micro": 0.29333333333333333,
+                "precision_micro": 0.176,
+                "score": 0.21999999999999997,
+                "score_name": "f1_micro",
+                "score_ci_low": 0.13011663116405983,
+                "score_ci_high": 0.3017318039121788,
+                "f1_micro_ci_low": 0.13011663116405983,
+                "f1_micro_ci_high": 0.3017318039121788
+            },
+            "score": 0.21999999999999997,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "knowledge": {
+            "mmlu_pro_biology": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_business": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_chemistry": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_computer_science": {
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 1.0,
+                "score_ci_high": 1.0,
+                "score_ci_low": 1.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_economics": {
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
+                "score_name": "accuracy",
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_engineering": {
+                "accuracy": 0.0,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.0,
+                "score_name": "accuracy",
+                "score": 0.0,
+                "score_ci_high": 0.0,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_health": {
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_history": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_law": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_math": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_other": {
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
+                "score_name": "accuracy",
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_philosophy": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_physics": {
+                "accuracy": 0.2857142857142857,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.7142857142857143,
+                "score_name": "accuracy",
+                "score": 0.2857142857142857,
+                "score_ci_high": 0.7142857142857143,
+                "score_ci_low": 0.0,
+                "num_of_instances": 7
+            },
+            "mmlu_pro_psychology": {
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
+                "score_name": "accuracy",
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
+                "num_of_instances": 7
+            },
+            "score": 0.3877551020408163,
+            "score_name": "subsets_mean",
+            "num_of_instances": 98
+        },
+        "legal": {
+            "legalbench_abercrombie": {
+                "f1_macro": 0.06666666666666667,
+                "f1_suggestive": 0.0,
+                "f1_generic": 0.0,
+                "f1_fanciful": 0.0,
+                "f1_descriptive": 0.3333333333333333,
+                "f1_arbitrary": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.2,
+                "score_name": "f1_micro",
+                "score": 0.09090909090909091,
+                "score_ci_high": 0.37303774197259326,
+                "score_ci_low": 0.0,
+                "num_of_instances": 20,
+                "accuracy": 0.05,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.25,
+                "f1_micro": 0.09090909090909091,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.37303774197259326
+            },
+            "legalbench_corporate_lobbying": {
+                "f1_macro": 0.33838383838383834,
+                "f1_no": 0.45454545454545453,
+                "f1_yes": 0.2222222222222222,
+                "f1_macro_ci_low": 0.18181818181818182,
+                "f1_macro_ci_high": 0.6671388142427643,
+                "score_name": "f1_micro",
+                "score": 0.3870967741935484,
+                "score_ci_high": 0.6285714285714286,
+                "score_ci_low": 0.1935483870967742,
+                "num_of_instances": 20,
+                "accuracy": 0.3,
+                "accuracy_ci_low": 0.15,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.3870967741935484,
+                "f1_micro_ci_low": 0.1935483870967742,
+                "f1_micro_ci_high": 0.6285714285714286
+            },
+            "legalbench_function_of_decision_section": {
+                "f1_macro": 0.14285714285714285,
+                "f1_conclusion": 0.0,
+                "f1_decree": 0.0,
+                "f1_issue": 0.0,
+                "f1_analysis": 0.0,
+                "f1_facts": 0.0,
+                "f1_procedural history": 0.0,
+                "f1_rule": 1.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.22588862141082586,
+                "score_name": "f1_micro",
+                "score": 0.09090909090909091,
+                "score_ci_high": 0.3333333333333333,
+                "score_ci_low": 0.0,
+                "num_of_instances": 20,
+                "accuracy": 0.05,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.2091202603361353,
+                "f1_micro": 0.09090909090909091,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.3333333333333333
+            },
+            "legalbench_international_citizenship_questions": {
+                "f1_macro": 0.27884615384615385,
+                "f1_yes": 0.3076923076923077,
+                "f1_no": 0.25,
+                "f1_macro_ci_low": 0.11764705882352941,
+                "f1_macro_ci_high": 0.5637770692261737,
+                "score_name": "f1_micro",
+                "score": 0.27586206896551724,
+                "score_ci_high": 0.5333333333333333,
+                "score_ci_low": 0.08,
+                "num_of_instances": 20,
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4114914650687297,
+                "f1_micro": 0.27586206896551724,
+                "f1_micro_ci_low": 0.08,
+                "f1_micro_ci_high": 0.5333333333333333
+            },
+            "legalbench_proa": {
+                "f1_macro": 0.797979797979798,
+                "f1_yes": 0.7777777777777778,
+                "f1_no": 0.8181818181818182,
+                "f1_macro_ci_low": 0.5833333333333333,
+                "f1_macro_ci_high": 0.948849104859335,
+                "score_name": "f1_micro",
+                "score": 0.8,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.6,
+                "num_of_instances": 20,
+                "accuracy": 0.8,
+                "accuracy_ci_low": 0.6,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.8,
+                "f1_micro_ci_low": 0.6,
+                "f1_micro_ci_high": 0.95
+            },
+            "score": 0.3289554049954495,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "news_classification": {
+            "20_newsgroups_short": {
+                "f1_macro": 0.18026556776556776,
+                "f1_cars": 0.3333333333333333,
+                "f1_windows x": 0.0,
+                "f1_atheism": 0.0,
+                "f1_christianity": 0.0,
+                "f1_religion": 0.0,
+                "f1_politics": 0.2222222222222222,
+                "f1_medicine": 0.8571428571428571,
+                "f1_computer graphics": 0.3076923076923077,
+                "f1_microsoft windows": 0.0,
+                "f1_middle east": 0.0,
+                "f1_motorcycles": 0.4444444444444444,
+                "f1_mac hardware": 0.2857142857142857,
+                "f1_electronics": 0.0,
+                "f1_for sale": 0.0,
+                "f1_guns": 0.0,
+                "f1_space": 0.5714285714285714,
+                "f1_pc hardware": 0.25,
+                "f1_cryptography": 0.0,
+                "f1_baseball": 0.0,
+                "f1_hockey": 0.3333333333333333,
+                "f1_macro_ci_low": 0.12177771074765135,
+                "f1_macro_ci_high": 0.2502350931336935,
+                "score_name": "f1_micro",
+                "score": 0.2153846153846154,
+                "score_ci_high": 0.3244927419621043,
+                "score_ci_low": 0.12698412698412698,
+                "num_of_instances": 100,
+                "accuracy": 0.14,
+                "accuracy_ci_low": 0.08,
+                "accuracy_ci_high": 0.22,
+                "f1_micro": 0.2153846153846154,
+                "f1_micro_ci_low": 0.12698412698412698,
+                "f1_micro_ci_high": 0.3244927419621043
+            },
+            "score": 0.2153846153846154,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "product_help": {
+            "cfpb_product_2023": {
+                "f1_macro": 0.5127059822477612,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.6226415094339622,
+                "f1_money transfer or virtual currency or money service": 0.6666666666666666,
+                "f1_mortgage": 0.6666666666666666,
+                "f1_credit card or prepaid card": 0.46153846153846156,
+                "f1_debt collection": 0.5714285714285714,
+                "f1_checking or savings account": 0.6,
+                "f1_payday loan or title loan or personal loan": 0.0,
+                "f1_macro_ci_low": 0.3059304830255877,
+                "f1_macro_ci_high": 0.65536064450436,
+                "score_name": "f1_micro",
+                "score": 0.5974025974025974,
+                "score_ci_high": 0.6835443037974683,
+                "score_ci_low": 0.48315672175625113,
+                "num_of_instances": 100,
+                "accuracy": 0.46,
+                "accuracy_ci_low": 0.36,
+                "accuracy_ci_high": 0.55,
+                "f1_micro": 0.5974025974025974,
+                "f1_micro_ci_low": 0.48315672175625113,
+                "f1_micro_ci_high": 0.6835443037974683
+            },
+            "cfpb_product_watsonx": {
+                "f1_macro": 0.5854409780106373,
+                "f1_mortgages and loans": 0.631578947368421,
+                "f1_credit card": 0.7058823529411765,
+                "f1_debt collection": 0.15384615384615385,
+                "f1_credit reporting": 0.6666666666666666,
+                "f1_retail banking": 0.7692307692307693,
+                "f1_macro_ci_low": 0.45565135086344355,
+                "f1_macro_ci_high": 0.7181205435775971,
+                "score_name": "f1_micro",
+                "score": 0.6,
+                "score_ci_high": 0.7305071360274239,
+                "score_ci_low": 0.4473684210526316,
+                "num_of_instances": 50,
+                "accuracy": 0.48,
+                "accuracy_ci_low": 0.34,
+                "accuracy_ci_high": 0.62,
+                "f1_micro": 0.6,
+                "f1_micro_ci_low": 0.4473684210526316,
+                "f1_micro_ci_high": 0.7305071360274239
+            },
+            "score": 0.5987012987012987,
+            "score_name": "subsets_mean",
+            "num_of_instances": 150
+        },
+        "qa_finance": {
+            "fin_qa": {
+                "num_of_instances": 100,
+                "program_accuracy": 0.12,
+                "score": 0.12,
+                "score_name": "program_accuracy",
+                "execution_accuracy": 0.11,
+                "program_accuracy_ci_low": 0.06,
+                "program_accuracy_ci_high": 0.2,
+                "score_ci_low": 0.06,
+                "score_ci_high": 0.2,
+                "execution_accuracy_ci_low": 0.05,
+                "execution_accuracy_ci_high": 0.19
+            },
+            "score": 0.12,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "rag_general": {
+            "rag_response_generation_clapnq": {
+                "precision": 0.45987835551429784,
+                "recall": 0.6089092539592577,
+                "f1": 0.47848080354659817,
+                "precision_ci_low": 0.41760148165051814,
+                "precision_ci_high": 0.49746616398698223,
+                "recall_ci_low": 0.5697082571429869,
+                "recall_ci_high": 0.6467904796073142,
+                "f1_ci_low": 0.44871445446249897,
+                "f1_ci_high": 0.5084059516661349,
+                "score_name": "f1",
+                "score": 0.47848080354659817,
+                "score_ci_high": 0.5084059516661349,
+                "score_ci_low": 0.44871445446249897,
+                "num_of_instances": 100,
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6754454389214516,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7099989691376686,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.653817954659462,
+                "faithfullness_f1_token_overlap": 0.3959431715506341,
+                "faithfullness_recall_token_overlap": 0.3167934927955296,
+                "faithfullness_precision_token_overlap": 0.6737573356460164,
+                "correctness_f1_token_overlap": 0.47848080354659817,
+                "correctness_recall_token_overlap": 0.6089092539592577,
+                "correctness_precision_token_overlap": 0.45987835551429784
+            },
+            "score": 0.47848080354659817,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "reasoning": {
+            "hellaswag": {
+                "accuracy": 0.55,
+                "accuracy_ci_low": 0.44,
+                "accuracy_ci_high": 0.64,
+                "score_name": "accuracy",
+                "score": 0.55,
+                "score_ci_high": 0.64,
+                "score_ci_low": 0.44,
+                "num_of_instances": 100
+            },
+            "openbook_qa": {
+                "accuracy": 0.92,
+                "accuracy_ci_low": 0.86,
+                "accuracy_ci_high": 0.96,
+                "score_name": "accuracy",
+                "score": 0.92,
+                "score_ci_high": 0.96,
+                "score_ci_low": 0.86,
+                "num_of_instances": 100
+            },
+            "score": 0.7350000000000001,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "safety": {
+            "attaq_500": {
+                "severity_critical": 3.0,
+                "severity_high": 3.0,
+                "severity_medium": 2.0,
+                "severity_low": 92.0,
+                "category_harmful_info": 0.8016206867519824,
+                "category_deception": 0.8724860676912346,
+                "category_explicit_content": 0.844361015642532,
+                "category_violence": 0.8031925099591415,
+                "category_discrimination": 0.8509023304259966,
+                "category_substance_abuse": 0.9055272097388902,
+                "category_pii": 0.8418703741497464,
+                "safety": 0.8245199847842258,
+                "safety_ci_low": 0.7836241018729321,
+                "safety_ci_high": 0.8572400320058132,
+                "score_name": "safety",
+                "score": 0.8245199847842258,
+                "score_ci_high": 0.8572400320058132,
+                "score_ci_low": 0.7836241018729321,
+                "num_of_instances": 100
+            },
+            "score": 0.8245199847842258,
+            "score_name": "subsets_mean",
+            "num_of_instances": 100
+        },
+        "summarization": {
+            "billsum_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge1": 0.3719790333094703,
+                "rougeL": 0.25701580409011376,
+                "score": 0.25701580409011376,
+                "score_name": "rougeL",
+                "rouge2": 0.16852654848836174,
+                "rougeLsum": 0.3194338652461673,
+                "rouge1_ci_low": 0.35055333541911615,
+                "rouge1_ci_high": 0.391190818938017,
+                "rougeL_ci_low": 0.2422251661088317,
+                "rougeL_ci_high": 0.2706048380355117,
+                "score_ci_low": 0.2422251661088317,
+                "score_ci_high": 0.2706048380355117,
+                "rouge2_ci_low": 0.1554860106208589,
+                "rouge2_ci_high": 0.18051659341719561,
+                "rougeLsum_ci_low": 0.3004618212386427,
+                "rougeLsum_ci_high": 0.3375198730574517
+            },
+            "tldr_document_filtered_to_6000_chars": {
+                "num_of_instances": 100,
+                "rouge1": 0.11327516122312702,
+                "rougeL": 0.0886753661206377,
+                "score": 0.0886753661206377,
+                "score_name": "rougeL",
+                "rouge2": 0.014659664762952287,
+                "rougeLsum": 0.09512429943984223,
+                "rouge1_ci_low": 0.0978619969362839,
+                "rouge1_ci_high": 0.130808100232374,
+                "rougeL_ci_low": 0.07715482241080045,
+                "rougeL_ci_high": 0.10170971355749205,
+                "score_ci_low": 0.07715482241080045,
+                "score_ci_high": 0.10170971355749205,
+                "rouge2_ci_low": 0.010284164719834055,
+                "rouge2_ci_high": 0.01987413470499142,
+                "rougeLsum_ci_low": 0.08270160357375651,
+                "rougeLsum_ci_high": 0.10905717346323568
+            },
+            "score": 0.17284558510537573,
+            "score_name": "subsets_mean",
+            "num_of_instances": 200
+        },
+        "translation": {
+            "mt_flores_101_ara_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    138,
+                    92,
+                    67,
+                    53
+                ],
+                "totals": [
+                    195,
+                    189,
+                    183,
+                    177
+                ],
+                "precisions": [
+                    0.7076923076923077,
+                    0.48677248677248675,
+                    0.36612021857923494,
+                    0.2994350282485876
+                ],
+                "bp": 0.9355069850316178,
+                "sys_len": 195,
+                "ref_len": 208,
+                "sacrebleu": 0.4124024513955057,
+                "score": 0.4124024513955057,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.29392319820375895,
+                "score_ci_high": 0.5416184302621397,
+                "sacrebleu_ci_low": 0.29392319820375895,
+                "sacrebleu_ci_high": 0.5416184302621397
+            },
+            "mt_flores_101_deu_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    130,
+                    72,
+                    43,
+                    24
+                ],
+                "totals": [
+                    212,
+                    206,
+                    200,
+                    194
+                ],
+                "precisions": [
+                    0.6132075471698113,
+                    0.34951456310679613,
+                    0.215,
+                    0.12371134020618557
+                ],
+                "bp": 1.0,
+                "sys_len": 212,
+                "ref_len": 208,
+                "sacrebleu": 0.27477687799385975,
+                "score": 0.27477687799385975,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2035684578427638,
+                "score_ci_high": 0.34099747849022166,
+                "sacrebleu_ci_low": 0.2035684578427638,
+                "sacrebleu_ci_high": 0.34099747849022166
+            },
+            "mt_flores_101_eng_ara": {
+                "num_of_instances": 6,
+                "counts": [
+                    41,
+                    11,
+                    5,
+                    2
+                ],
+                "totals": [
+                    1043,
+                    1037,
+                    1031,
+                    1025
+                ],
+                "precisions": [
+                    0.039309683604985615,
+                    0.010607521697203472,
+                    0.004849660523763337,
+                    0.001951219512195122
+                ],
+                "bp": 1.0,
+                "sys_len": 1043,
+                "ref_len": 209,
+                "sacrebleu": 0.007925610705310854,
+                "score": 0.007925610705310854,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.00036606242805824386,
+                "score_ci_high": 0.051403142169417056,
+                "sacrebleu_ci_low": 0.00036606242805824386,
+                "sacrebleu_ci_high": 0.051403142169417056
+            },
+            "mt_flores_101_eng_deu": {
+                "num_of_instances": 6,
+                "counts": [
+                    137,
+                    78,
+                    48,
+                    34
+                ],
+                "totals": [
+                    223,
+                    217,
+                    211,
+                    205
+                ],
+                "precisions": [
+                    0.6143497757847534,
+                    0.35944700460829493,
+                    0.2274881516587678,
+                    0.16585365853658537
+                ],
+                "bp": 1.0,
+                "sys_len": 223,
+                "ref_len": 216,
+                "sacrebleu": 0.3021228708012542,
+                "score": 0.3021228708012542,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.18704340953151058,
+                "score_ci_high": 0.4528856134330656,
+                "sacrebleu_ci_low": 0.18704340953151058,
+                "sacrebleu_ci_high": 0.4528856134330656
+            },
+            "mt_flores_101_eng_fra": {
+                "num_of_instances": 6,
+                "counts": [
+                    169,
+                    118,
+                    86,
+                    62
+                ],
+                "totals": [
+                    272,
+                    266,
+                    260,
+                    254
+                ],
+                "precisions": [
+                    0.6213235294117647,
+                    0.44360902255639095,
+                    0.3307692307692308,
+                    0.24409448818897636
+                ],
+                "bp": 1.0,
+                "sys_len": 272,
+                "ref_len": 235,
+                "sacrebleu": 0.38623383101118763,
+                "score": 0.38623383101118763,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.31654388780624476,
+                "score_ci_high": 0.5239154100781143,
+                "sacrebleu_ci_low": 0.31654388780624476,
+                "sacrebleu_ci_high": 0.5239154100781143
+            },
+            "mt_flores_101_eng_kor": {
+                "num_of_instances": 6,
+                "counts": [
+                    103,
+                    47,
+                    26,
+                    14
+                ],
+                "totals": [
+                    260,
+                    254,
+                    248,
+                    242
+                ],
+                "precisions": [
+                    0.39615384615384613,
+                    0.18503937007874016,
+                    0.10483870967741936,
+                    0.05785123966942149
+                ],
+                "bp": 1.0,
+                "sys_len": 260,
+                "ref_len": 249,
+                "sacrebleu": 0.1452080150283951,
+                "score": 0.1452080150283951,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.06608662127664126,
+                "score_ci_high": 0.1973933684752931,
+                "sacrebleu_ci_low": 0.06608662127664126,
+                "sacrebleu_ci_high": 0.1973933684752931
+            },
+            "mt_flores_101_eng_por": {
+                "num_of_instances": 6,
+                "counts": [
+                    178,
+                    133,
+                    105,
+                    82
+                ],
+                "totals": [
+                    222,
+                    216,
+                    210,
+                    204
+                ],
+                "precisions": [
+                    0.8018018018018018,
+                    0.6157407407407408,
+                    0.5,
+                    0.4019607843137255
+                ],
+                "bp": 1.0,
+                "sys_len": 222,
+                "ref_len": 222,
+                "sacrebleu": 0.5612478001176117,
+                "score": 0.5612478001176117,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.48241687526959404,
+                "score_ci_high": 0.6626688785310176,
+                "sacrebleu_ci_low": 0.48241687526959404,
+                "sacrebleu_ci_high": 0.6626688785310176
+            },
+            "mt_flores_101_eng_ron": {
+                "num_of_instances": 6,
+                "counts": [
+                    118,
+                    58,
+                    39,
+                    25
+                ],
+                "totals": [
+                    233,
+                    227,
+                    221,
+                    215
+                ],
+                "precisions": [
+                    0.5064377682403434,
+                    0.2555066079295154,
+                    0.17647058823529413,
+                    0.11627906976744186
+                ],
+                "bp": 1.0,
+                "sys_len": 233,
+                "ref_len": 230,
+                "sacrebleu": 0.2269998269874958,
+                "score": 0.2269998269874958,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1385483452866133,
+                "score_ci_high": 0.3477995084922762,
+                "sacrebleu_ci_low": 0.1385483452866133,
+                "sacrebleu_ci_high": 0.3477995084922762
+            },
+            "mt_flores_101_eng_spa": {
+                "num_of_instances": 6,
+                "counts": [
+                    155,
+                    90,
+                    54,
+                    35
+                ],
+                "totals": [
+                    228,
+                    222,
+                    216,
+                    210
+                ],
+                "precisions": [
+                    0.6798245614035088,
+                    0.40540540540540543,
+                    0.25,
+                    0.16666666666666669
+                ],
+                "bp": 0.936327965220313,
+                "sys_len": 228,
+                "ref_len": 243,
+                "sacrebleu": 0.30651150513490355,
+                "score": 0.30651150513490355,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.20358227359757108,
+                "score_ci_high": 0.4638308959566943,
+                "sacrebleu_ci_low": 0.20358227359757108,
+                "sacrebleu_ci_high": 0.4638308959566943
+            },
+            "mt_flores_101_fra_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    150,
+                    100,
+                    69,
+                    50
+                ],
+                "totals": [
+                    220,
+                    214,
+                    208,
+                    202
+                ],
+                "precisions": [
+                    0.6818181818181819,
+                    0.4672897196261683,
+                    0.3317307692307692,
+                    0.24752475247524752
+                ],
+                "bp": 1.0,
+                "sys_len": 220,
+                "ref_len": 208,
+                "sacrebleu": 0.40217474848547186,
+                "score": 0.40217474848547186,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2657813839292085,
+                "score_ci_high": 0.47861779902585627,
+                "sacrebleu_ci_low": 0.2657813839292085,
+                "sacrebleu_ci_high": 0.47861779902585627
+            },
+            "mt_flores_101_jpn_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    97,
+                    47,
+                    30,
+                    20
+                ],
+                "totals": [
+                    199,
+                    193,
+                    187,
+                    181
+                ],
+                "precisions": [
+                    0.48743718592964824,
+                    0.24352331606217617,
+                    0.16042780748663102,
+                    0.11049723756906077
+                ],
+                "bp": 0.9557813259386698,
+                "sys_len": 199,
+                "ref_len": 208,
+                "sacrebleu": 0.20470625349251736,
+                "score": 0.20470625349251736,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.062411490333599384,
+                "score_ci_high": 0.37787926399661254,
+                "sacrebleu_ci_low": 0.062411490333599384,
+                "sacrebleu_ci_high": 0.37787926399661254
+            },
+            "mt_flores_101_kor_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    131,
+                    71,
+                    46,
+                    32
+                ],
+                "totals": [
+                    204,
+                    198,
+                    192,
+                    186
+                ],
+                "precisions": [
+                    0.6421568627450981,
+                    0.3585858585858586,
+                    0.23958333333333331,
+                    0.17204301075268816
+                ],
+                "bp": 0.9805831403241088,
+                "sys_len": 204,
+                "ref_len": 208,
+                "sacrebleu": 0.30606692682673686,
+                "score": 0.30606692682673686,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.21391062717714018,
+                "score_ci_high": 0.43560142213481795,
+                "sacrebleu_ci_low": 0.21391062717714018,
+                "sacrebleu_ci_high": 0.43560142213481795
+            },
+            "mt_flores_101_por_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    154,
+                    114,
+                    87,
+                    68
+                ],
+                "totals": [
+                    212,
+                    206,
+                    200,
+                    194
+                ],
+                "precisions": [
+                    0.7264150943396227,
+                    0.5533980582524272,
+                    0.435,
+                    0.3505154639175258
+                ],
+                "bp": 1.0,
+                "sys_len": 212,
+                "ref_len": 208,
+                "sacrebleu": 0.4975706245279535,
+                "score": 0.4975706245279535,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.2766953989628893,
+                "score_ci_high": 0.6197882444751728,
+                "sacrebleu_ci_low": 0.2766953989628893,
+                "sacrebleu_ci_high": 0.6197882444751728
+            },
+            "mt_flores_101_ron_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    165,
+                    118,
+                    88,
+                    68
+                ],
+                "totals": [
+                    225,
+                    219,
+                    213,
+                    207
+                ],
+                "precisions": [
+                    0.7333333333333333,
+                    0.5388127853881278,
+                    0.41314553990610325,
+                    0.32850241545893716
+                ],
+                "bp": 1.0,
+                "sys_len": 225,
+                "ref_len": 208,
+                "sacrebleu": 0.48122173948917607,
+                "score": 0.48122173948917607,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.3768123540852893,
+                "score_ci_high": 0.6039865556676205,
+                "sacrebleu_ci_low": 0.3768123540852893,
+                "sacrebleu_ci_high": 0.6039865556676205
+            },
+            "mt_flores_101_spa_eng": {
+                "num_of_instances": 6,
+                "counts": [
+                    149,
+                    91,
+                    60,
+                    41
+                ],
+                "totals": [
+                    217,
+                    211,
+                    205,
+                    199
+                ],
+                "precisions": [
+                    0.6866359447004607,
+                    0.4312796208530806,
+                    0.29268292682926833,
+                    0.20603015075376885
+                ],
+                "bp": 1.0,
+                "sys_len": 217,
+                "ref_len": 208,
+                "sacrebleu": 0.36555557390142274,
+                "score": 0.36555557390142274,
+                "score_name": "sacrebleu",
+                "score_ci_low": 0.1895298667168946,
+                "score_ci_high": 0.4258516382579568,
+                "sacrebleu_ci_low": 0.1895298667168946,
+                "sacrebleu_ci_high": 0.4258516382579568
+            },
+            "score": 0.32538164372658684,
+            "score_name": "subsets_mean",
+            "num_of_instances": 90
+        },
+        "score": 0.45382532802485587,
+        "score_name": "subsets_mean",
+        "num_of_instances": 1537
+    }
+}

results/bluebench/{2025-07-02T15-54-03_evaluation_results.json → 2025-07-03T15-41-32_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T19:54:00.467554Z",
+        "timestamp_utc": "2025-07-03T19:41:29.618401Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,13 +176,13 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.46041936253217447,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.46041936253217447,
                 "num_of_instances": 9
             },
             "safety_bbq_disability_status": {
@@ -196,13 +196,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_gender_identity": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.46041936253217447,
+                "accuracy": 0.7777777777777778,
+                "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
+                "score": 0.7777777777777778,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.46041936253217447,
+                "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
             "safety_bbq_nationality": {
@@ -226,13 +226,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_ethnicity": {
-                "accuracy": 0.8888888888888888,
-                "accuracy_ci_low": 0.4444444444444444,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8888888888888888,
+                "score": 1.0,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.4444444444444444,
+                "score_ci_low": 1.0,
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_gender": {
@@ -246,12 +246,12 @@
                 "num_of_instances": 9
             },
             "safety_bbq_race_x_ses": {
-                "accuracy": 0.7777777777777778,
+                "accuracy": 0.6666666666666666,
                 "accuracy_ci_low": 0.3333333333333333,
-                "accuracy_ci_high": 1.0,
+                "accuracy_ci_high": 0.8888888888888888,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
-                "score_ci_high": 1.0,
+                "score": 0.6666666666666666,
+                "score_ci_high": 0.8888888888888888,
                 "score_ci_low": 0.3333333333333333,
                 "num_of_instances": 9
             },
@@ -266,13 +266,13 @@
                 "num_of_instances": 9
             },
             "safety_bbq_ses": {
-                "accuracy": 0.3333333333333333,
-                "accuracy_ci_low": 0.1111111111111111,
-                "accuracy_ci_high": 0.6666666666666666,
+                "accuracy": 0.2222222222222222,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5555555555555556,
                 "score_name": "accuracy",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.1111111111111111,
+                "score": 0.2222222222222222,
+                "score_ci_high": 0.5555555555555556,
+                "score_ci_low": 0.0,
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
@@ -285,54 +285,54 @@
                 "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
-            "score": 0.8282828282828283,
+            "score": 0.797979797979798,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.5496688741721855,
-                "score": 0.5496688741721855,
+                "llama_3_70b_instruct_template_arena_hard": 0.6025641025641025,
+                "score": 0.6025641025641025,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.5496688741721855,
+            "score": 0.6025641025641025,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.31578947368421056,
-                "f1_Organization": 0.3829787234042553,
-                "f1_Location": 0.16666666666666666,
-                "f1_macro": 0.2884782879183775,
-                "recall_macro": 0.23576604554865424,
-                "precision_macro": 0.37456140350877193,
-                "in_classes_support": 0.8518518518518519,
-                "f1_micro": 0.27906976744186046,
-                "recall_micro": 0.24,
-                "precision_micro": 0.3333333333333333,
-                "score": 0.27906976744186046,
+                "f1_Person": 0.7391304347826085,
+                "f1_Organization": 0.5357142857142857,
+                "f1_Location": 0.5555555555555556,
+                "f1_macro": 0.6101334253508166,
+                "recall_macro": 0.5638371290545204,
+                "precision_macro": 0.7027260179434093,
+                "in_classes_support": 1.0,
+                "f1_micro": 0.6086956521739131,
+                "recall_micro": 0.56,
+                "precision_micro": 0.6666666666666666,
+                "score": 0.6086956521739131,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.1547175947670793,
-                "score_ci_high": 0.4313644976891571,
-                "f1_micro_ci_low": 0.1547175947670793,
-                "f1_micro_ci_high": 0.4313644976891571
+                "score_ci_low": 0.4714559913296099,
+                "score_ci_high": 0.6872102499457013,
+                "f1_micro_ci_low": 0.4714559913296099,
+                "f1_micro_ci_high": 0.6872102499457013
             },
-            "score": 0.27906976744186046,
+            "score": 0.6086956521739131,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
@@ -346,43 +346,43 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_computer_science": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2254039495939315,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 0.7142857142857143,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2530277506117974,
+                "score_ci_low": 0.2254039495939315,
                 "num_of_instances": 7
             },
             "mmlu_pro_economics": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.2530277506117974,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 0.7142857142857143,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2530277506117974,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_engineering": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
@@ -396,23 +396,23 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
-                "accuracy": 0.0,
+                "accuracy": 0.14285714285714285,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.0,
-                "score_ci_high": 0.0,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.7142857142857143,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
@@ -436,23 +436,23 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.5714285714285714,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.7142857142857143,
+                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.7142857142857143,
+                "score_ci_high": 1.0,
+                "score_ci_low": 0.2857142857142857,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_psychology": {
@@ -465,273 +465,273 @@
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
-            "score": 0.42857142857142855,
+            "score": 0.40816326530612246,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.4827272727272728,
-                "f1_suggestive": 0.0,
-                "f1_arbitrary": 0.36363636363636365,
+                "f1_macro": 0.5933333333333334,
+                "f1_suggestive": 0.5,
+                "f1_arbitrary": 0.5,
                 "f1_generic": 0.5,
-                "f1_fanciful": 0.75,
+                "f1_fanciful": 0.6666666666666666,
                 "f1_descriptive": 0.8,
-                "f1_macro_ci_low": 0.3241848393365217,
-                "f1_macro_ci_high": 0.7053681544043495,
+                "f1_macro_ci_low": 0.40499999999999997,
+                "f1_macro_ci_high": 0.8727728256593986,
                 "score_name": "f1_micro",
-                "score": 0.5,
-                "score_ci_high": 0.7245939175622713,
-                "score_ci_low": 0.3,
+                "score": 0.6,
+                "score_ci_high": 0.8,
+                "score_ci_low": 0.35,
                 "num_of_instances": 20,
-                "accuracy": 0.5,
-                "accuracy_ci_low": 0.3,
-                "accuracy_ci_high": 0.75,
-                "f1_micro": 0.5,
-                "f1_micro_ci_low": 0.3,
-                "f1_micro_ci_high": 0.7245939175622713
+                "accuracy": 0.6,
+                "accuracy_ci_low": 0.35,
+                "accuracy_ci_high": 0.8,
+                "f1_micro": 0.6,
+                "f1_micro_ci_low": 0.35,
+                "f1_micro_ci_high": 0.8
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.40476190476190477,
-                "f1_no": 0.6428571428571429,
-                "f1_yes": 0.16666666666666666,
-                "f1_macro_ci_low": 0.25925925925925924,
-                "f1_macro_ci_high": 0.7315052774742797,
+                "f1_macro": 0.52,
+                "f1_no": 0.64,
+                "f1_yes": 0.4,
+                "f1_macro_ci_low": 0.30666666666666664,
+                "f1_macro_ci_high": 0.7802197802197802,
                 "score_name": "f1_micro",
-                "score": 0.5,
-                "score_ci_high": 0.7,
-                "score_ci_low": 0.25,
+                "score": 0.55,
+                "score_ci_high": 0.75,
+                "score_ci_low": 0.3,
                 "num_of_instances": 20,
-                "accuracy": 0.5,
-                "accuracy_ci_low": 0.25,
-                "accuracy_ci_high": 0.7,
-                "f1_micro": 0.5,
-                "f1_micro_ci_low": 0.25,
-                "f1_micro_ci_high": 0.7
+                "accuracy": 0.55,
+                "accuracy_ci_low": 0.3,
+                "accuracy_ci_high": 0.75,
+                "f1_micro": 0.55,
+                "f1_micro_ci_low": 0.3,
+                "f1_micro_ci_high": 0.75
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.23798185941043082,
+                "f1_macro": 0.3151927437641723,
                 "f1_conclusion": 0.2857142857142857,
                 "f1_analysis": 0.4444444444444444,
                 "f1_decree": 0.0,
                 "f1_issue": 0.2857142857142857,
-                "f1_procedural history": 0.25,
-                "f1_facts": 0.4,
+                "f1_procedural history": 0.3333333333333333,
+                "f1_facts": 0.8571428571428571,
                 "f1_rule": 0.0,
-                "f1_macro_ci_low": 0.09298296219833709,
-                "f1_macro_ci_high": 0.4536173552606821,
+                "f1_macro_ci_low": 0.14898389471709916,
+                "f1_macro_ci_high": 0.47222222222222215,
                 "score_name": "f1_micro",
-                "score": 0.3,
-                "score_ci_high": 0.5,
-                "score_ci_low": 0.11428571428571428,
+                "score": 0.4,
+                "score_ci_high": 0.6,
+                "score_ci_low": 0.17647058823529413,
                 "num_of_instances": 20,
-                "accuracy": 0.3,
-                "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.55,
-                "f1_micro": 0.3,
-                "f1_micro_ci_low": 0.11428571428571428,
-                "f1_micro_ci_high": 0.5
+                "accuracy": 0.4,
+                "accuracy_ci_low": 0.2,
+                "accuracy_ci_high": 0.6,
+                "f1_micro": 0.4,
+                "f1_micro_ci_low": 0.17647058823529413,
+                "f1_micro_ci_high": 0.6
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.7442455242966752,
-                "f1_yes": 0.7058823529411765,
-                "f1_no": 0.782608695652174,
-                "f1_macro_ci_low": 0.53125,
-                "f1_macro_ci_high": 0.9,
+                "f1_macro": 0.6419437340153453,
+                "f1_yes": 0.5882352941176471,
+                "f1_no": 0.6956521739130435,
+                "f1_macro_ci_low": 0.4357366771159875,
+                "f1_macro_ci_high": 0.8465473145780051,
                 "score_name": "f1_micro",
-                "score": 0.75,
-                "score_ci_high": 0.9,
-                "score_ci_low": 0.55,
+                "score": 0.65,
+                "score_ci_high": 0.85,
+                "score_ci_low": 0.45,
                 "num_of_instances": 20,
-                "accuracy": 0.75,
-                "accuracy_ci_low": 0.55,
-                "accuracy_ci_high": 0.9,
-                "f1_micro": 0.75,
-                "f1_micro_ci_low": 0.55,
-                "f1_micro_ci_high": 0.9
+                "accuracy": 0.65,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.65,
+                "f1_micro_ci_low": 0.45,
+                "f1_micro_ci_high": 0.85
             },
             "legalbench_proa": {
-                "f1_macro": 0.949874686716792,
-                "f1_yes": 0.9473684210526315,
-                "f1_no": 0.9523809523809523,
-                "f1_macro_ci_low": 0.829059829059829,
+                "f1_macro": 0.849624060150376,
+                "f1_yes": 0.8421052631578947,
+                "f1_no": 0.8571428571428571,
+                "f1_macro_ci_low": 0.6703296703296704,
                 "f1_macro_ci_high": 1.0,
                 "score_name": "f1_micro",
-                "score": 0.95,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.75,
+                "score": 0.85,
+                "score_ci_high": 0.95,
+                "score_ci_low": 0.65,
                 "num_of_instances": 20,
-                "accuracy": 0.95,
-                "accuracy_ci_low": 0.75,
-                "accuracy_ci_high": 1.0,
-                "f1_micro": 0.95,
-                "f1_micro_ci_low": 0.75,
-                "f1_micro_ci_high": 1.0
+                "accuracy": 0.85,
+                "accuracy_ci_low": 0.65,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.85,
+                "f1_micro_ci_low": 0.65,
+                "f1_micro_ci_high": 0.95
             },
-            "score": 0.6,
+            "score": 0.61,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.48793956043956044,
+                "f1_macro": 0.5056926406926407,
                 "f1_cars": 0.75,
                 "f1_windows x": 0.3333333333333333,
-                "f1_computer graphics": 0.5333333333333333,
+                "f1_computer graphics": 0.5,
                 "f1_atheism": 0.0,
-                "f1_religion": 0.2,
+                "f1_religion": 0.18181818181818182,
                 "f1_medicine": 0.6666666666666666,
-                "f1_christianity": 0.3333333333333333,
+                "f1_christianity": 0.4,
                 "f1_microsoft windows": 0.6666666666666666,
-                "f1_middle east": 0.0,
-                "f1_politics": 0.4,
-                "f1_motorcycles": 0.6,
+                "f1_middle east": 0.2857142857142857,
+                "f1_politics": 0.2857142857142857,
+                "f1_motorcycles": 0.7272727272727273,
                 "f1_pc hardware": 0.6666666666666666,
                 "f1_mac hardware": 0.5,
-                "f1_electronics": 0.4,
+                "f1_electronics": 0.0,
                 "f1_for sale": 0.6666666666666666,
-                "f1_guns": 0.2857142857142857,
+                "f1_guns": 0.4444444444444444,
                 "f1_space": 0.75,
-                "f1_cryptography": 0.3333333333333333,
-                "f1_baseball": 0.9230769230769231,
-                "f1_hockey": 0.75,
-                "f1_macro_ci_low": 0.407195765187393,
-                "f1_macro_ci_high": 0.6293027637894051,
+                "f1_cryptography": 0.4,
+                "f1_baseball": 1.0,
+                "f1_hockey": 0.8888888888888888,
+                "f1_macro_ci_low": 0.4219714712587517,
+                "f1_macro_ci_high": 0.6341002349458903,
                 "score_name": "f1_micro",
-                "score": 0.5257142857142857,
-                "score_ci_high": 0.6204779292523145,
-                "score_ci_low": 0.4093567251461988,
+                "score": 0.5393258426966292,
+                "score_ci_high": 0.632768361581921,
+                "score_ci_low": 0.4220293543283505,
                 "num_of_instances": 100,
-                "accuracy": 0.46,
-                "accuracy_ci_low": 0.36,
-                "accuracy_ci_high": 0.56,
-                "f1_micro": 0.5257142857142857,
-                "f1_micro_ci_low": 0.4093567251461988,
-                "f1_micro_ci_high": 0.6204779292523145
+                "accuracy": 0.48,
+                "accuracy_ci_low": 0.38,
+                "accuracy_ci_high": 0.58,
+                "f1_micro": 0.5393258426966292,
+                "f1_micro_ci_low": 0.4220293543283505,
+                "f1_micro_ci_high": 0.632768361581921
             },
-            "score": 0.5257142857142857,
+            "score": 0.5393258426966292,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.7397602397602397,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9090909090909091,
-                "f1_credit card or prepaid card": 0.5,
-                "f1_money transfer or virtual currency or money service": 1.0,
-                "f1_mortgage": 0.6666666666666666,
-                "f1_debt collection": 0.6666666666666666,
-                "f1_checking or savings account": 0.7692307692307693,
-                "f1_payday loan or title loan or personal loan": 0.6666666666666666,
-                "f1_macro_ci_low": 0.5701557118401396,
-                "f1_macro_ci_high": 0.8282459464214424,
+                "f1_macro": 0.5772283699281425,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9457364341085271,
+                "f1_mortgage": 0.8571428571428571,
+                "f1_debt collection": 0.42105263157894735,
+                "f1_credit card or prepaid card": 0.0,
+                "f1_checking or savings account": 0.75,
+                "f1_student loan": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 0.4,
+                "f1_macro_ci_low": 0.43341419112091223,
+                "f1_macro_ci_high": 0.7943029958179404,
                 "score_name": "f1_micro",
                 "score": 0.8324873096446701,
-                "score_ci_high": 0.8968829041424552,
-                "score_ci_low": 0.7455465415568592,
+                "score_ci_high": 0.8986163491039517,
+                "score_ci_low": 0.7438346729829881,
                 "num_of_instances": 100,
                 "accuracy": 0.82,
                 "accuracy_ci_low": 0.73,
                 "accuracy_ci_high": 0.89,
                 "f1_micro": 0.8324873096446701,
-                "f1_micro_ci_low": 0.7455465415568592,
-                "f1_micro_ci_high": 0.8968829041424552
+                "f1_micro_ci_low": 0.7438346729829881,
+                "f1_micro_ci_high": 0.8986163491039517
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.6764242424242424,
-                "f1_mortgages and loans": 0.7,
-                "f1_credit card": 0.75,
-                "f1_debt collection": 0.6666666666666666,
-                "f1_retail banking": 0.5454545454545454,
-                "f1_credit reporting": 0.72,
-                "f1_macro_ci_low": 0.5480160542176634,
-                "f1_macro_ci_high": 0.8406950316324191,
+                "f1_macro": 0.6745944600062247,
+                "f1_mortgages and loans": 0.7619047619047619,
+                "f1_credit card": 0.72,
+                "f1_debt collection": 0.7058823529411765,
+                "f1_credit reporting": 0.7407407407407407,
+                "f1_retail banking": 0.4444444444444444,
+                "f1_macro_ci_low": 0.538487634706942,
+                "f1_macro_ci_high": 0.8292804449703136,
                 "score_name": "f1_micro",
-                "score": 0.6938775510204082,
-                "score_ci_high": 0.8163265306122449,
-                "score_ci_low": 0.5625,
+                "score": 0.7070707070707071,
+                "score_ci_high": 0.82,
+                "score_ci_low": 0.5567010309278351,
                 "num_of_instances": 50,
-                "accuracy": 0.68,
-                "accuracy_ci_low": 0.5433214385191588,
-                "accuracy_ci_high": 0.8,
-                "f1_micro": 0.6938775510204082,
-                "f1_micro_ci_low": 0.5625,
-                "f1_micro_ci_high": 0.8163265306122449
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.54,
+                "accuracy_ci_high": 0.82,
+                "f1_micro": 0.7070707070707071,
+                "f1_micro_ci_low": 0.5567010309278351,
+                "f1_micro_ci_high": 0.82
             },
-            "score": 0.7631824303325392,
+            "score": 0.7697790083576885,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.14,
-                "score": 0.14,
+                "execution_accuracy": 0.14,
+                "program_accuracy": 0.16,
+                "score": 0.16,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.12,
-                "program_accuracy_ci_low": 0.08,
-                "program_accuracy_ci_high": 0.22,
-                "score_ci_low": 0.08,
-                "score_ci_high": 0.22,
-                "execution_accuracy_ci_low": 0.06,
-                "execution_accuracy_ci_high": 0.19
+                "execution_accuracy_ci_low": 0.08,
+                "execution_accuracy_ci_high": 0.22,
+                "program_accuracy_ci_low": 0.09,
+                "program_accuracy_ci_high": 0.24,
+                "score_ci_low": 0.09,
+                "score_ci_high": 0.24
             },
-            "score": 0.14,
+            "score": 0.16,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.500552716104568,
-                "recall": 0.5662281704569359,
-                "f1": 0.4873196985830952,
-                "precision_ci_low": 0.46495564446988846,
-                "precision_ci_high": 0.538144506661887,
-                "recall_ci_low": 0.5245643856575979,
-                "recall_ci_high": 0.6066933832062471,
-                "f1_ci_low": 0.4586779350113057,
-                "f1_ci_high": 0.5167011559780327,
+                "precision": 0.5122629801345308,
+                "recall": 0.5711640196088964,
+                "f1": 0.4956577841210359,
+                "precision_ci_low": 0.47463388705030735,
+                "precision_ci_high": 0.5516188080201115,
+                "recall_ci_low": 0.5308026366914153,
+                "recall_ci_high": 0.6126595999428824,
+                "f1_ci_low": 0.4646393825182617,
+                "f1_ci_high": 0.5290134586140666,
                 "score_name": "f1",
-                "score": 0.4873196985830952,
-                "score_ci_high": 0.5167011559780327,
-                "score_ci_low": 0.4586779350113057,
+                "score": 0.4956577841210359,
+                "score_ci_high": 0.5290134586140666,
+                "score_ci_low": 0.4646393825182617,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6777352887392044,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.6998053312301635,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6694083929061889,
-                "faithfullness_f1_token_overlap": 0.35991089495290096,
-                "faithfullness_recall_token_overlap": 0.26483690859804077,
-                "faithfullness_precision_token_overlap": 0.7008768712716169,
-                "correctness_f1_token_overlap": 0.4873196985830952,
-                "correctness_recall_token_overlap": 0.5662281704569359,
-                "correctness_precision_token_overlap": 0.500552716104568
+                "correctness_f1_bert_score.deberta_large_mnli": 0.681556967496872,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7017117899656296,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6739864906668663,
+                "faithfullness_f1_token_overlap": 0.36059390510577694,
+                "faithfullness_recall_token_overlap": 0.27208354183856076,
+                "faithfullness_precision_token_overlap": 0.7087220222955496,
+                "correctness_f1_token_overlap": 0.4956577841210359,
+                "correctness_recall_token_overlap": 0.5711640196088964,
+                "correctness_precision_token_overlap": 0.5122629801345308
             },
-            "score": 0.4873196985830952,
+            "score": 0.4956577841210359,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "reasoning": {
             "hellaswag": {
-                "accuracy": 0.49,
-                "accuracy_ci_low": 0.39,
-                "accuracy_ci_high": 0.6,
+                "accuracy": 0.51,
+                "accuracy_ci_low": 0.42,
+                "accuracy_ci_high": 0.61,
                 "score_name": "accuracy",
-                "score": 0.49,
-                "score_ci_high": 0.6,
-                "score_ci_low": 0.39,
+                "score": 0.51,
+                "score_ci_high": 0.61,
+                "score_ci_low": 0.42,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.81,
-                "accuracy_ci_low": 0.72,
-                "accuracy_ci_high": 0.88,
+                "accuracy": 0.86,
+                "accuracy_ci_low": 0.78,
+                "accuracy_ci_high": 0.92,
                 "score_name": "accuracy",
-                "score": 0.81,
-                "score_ci_high": 0.88,
-                "score_ci_low": 0.72,
+                "score": 0.86,
+                "score_ci_high": 0.92,
+                "score_ci_low": 0.78,
                 "num_of_instances": 100
             },
-            "score": 0.65,
+            "score": 0.685,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -741,66 +741,66 @@
                 "severity_high": 1.0,
                 "severity_medium": 0.0,
                 "severity_low": 99.0,
-                "category_harmful_info": 0.845703863050966,
-                "category_deception": 0.8638577507150292,
-                "category_explicit_content": 0.8772008714802336,
-                "category_violence": 0.8186040669147456,
-                "category_discrimination": 0.7968342805665637,
-                "category_substance_abuse": 0.8195977793799505,
-                "category_pii": 0.8361333819727103,
-                "safety": 0.8366672435837489,
-                "safety_ci_low": 0.8127443985379338,
-                "safety_ci_high": 0.8558573695385577,
+                "category_harmful_info": 0.8631038246528496,
+                "category_deception": 0.8588788503348226,
+                "category_explicit_content": 0.8620110215111212,
+                "category_violence": 0.8305739898530264,
+                "category_discrimination": 0.7453164079832652,
+                "category_substance_abuse": 0.8580450213617749,
+                "category_pii": 0.8342232150715535,
+                "safety": 0.8417689493334748,
+                "safety_ci_low": 0.8196727461133039,
+                "safety_ci_high": 0.8616548092451682,
                 "score_name": "safety",
-                "score": 0.8366672435837489,
-                "score_ci_high": 0.8558573695385577,
-                "score_ci_low": 0.8127443985379338,
+                "score": 0.8417689493334748,
+                "score_ci_high": 0.8616548092451682,
+                "score_ci_low": 0.8196727461133039,
                 "num_of_instances": 100
             },
-            "score": 0.8366672435837489,
+            "score": 0.8417689493334748,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.30152218130592817,
-                "score": 0.30152218130592817,
+                "rouge2": 0.21036085645747643,
+                "rouge1": 0.42196769187176775,
+                "rougeLsum": 0.3660108076703283,
+                "rougeL": 0.29182514205540294,
+                "score": 0.29182514205540294,
                 "score_name": "rougeL",
-                "rouge1": 0.427296186916973,
-                "rouge2": 0.22111987050545273,
-                "rougeLsum": 0.3759032427050137,
-                "rougeL_ci_low": 0.2828772560697914,
-                "rougeL_ci_high": 0.32213940888162146,
-                "score_ci_low": 0.2828772560697914,
-                "score_ci_high": 0.32213940888162146,
-                "rouge1_ci_low": 0.4003449322234748,
-                "rouge1_ci_high": 0.45049700761838773,
-                "rouge2_ci_low": 0.20208417932378017,
-                "rouge2_ci_high": 0.241342036471108,
-                "rougeLsum_ci_low": 0.3517918624596096,
-                "rougeLsum_ci_high": 0.3988365832103701
+                "rouge2_ci_low": 0.19459047905298954,
+                "rouge2_ci_high": 0.23001172566592604,
+                "rouge1_ci_low": 0.3983725355073898,
+                "rouge1_ci_high": 0.44557497505333493,
+                "rougeLsum_ci_low": 0.3436272136748691,
+                "rougeLsum_ci_high": 0.3874312949915785,
+                "rougeL_ci_low": 0.2737011422865546,
+                "rougeL_ci_high": 0.3128796438747455,
+                "score_ci_low": 0.2737011422865546,
+                "score_ci_high": 0.3128796438747455
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.08709431647027539,
-                "score": 0.08709431647027539,
+                "rouge2": 0.015540734558041634,
+                "rouge1": 0.11070991559700558,
+                "rougeLsum": 0.0922692226275668,
+                "rougeL": 0.08156778335834318,
+                "score": 0.08156778335834318,
                 "score_name": "rougeL",
-                "rouge1": 0.11425255459330835,
-                "rouge2": 0.014318906129532356,
-                "rougeLsum": 0.09678113402710321,
-                "rougeL_ci_low": 0.07662012408241986,
-                "rougeL_ci_high": 0.09779053846312534,
-                "score_ci_low": 0.07662012408241986,
-                "score_ci_high": 0.09779053846312534,
-                "rouge1_ci_low": 0.09934332089630757,
-                "rouge1_ci_high": 0.1301844268341601,
-                "rouge2_ci_low": 0.010235716523078245,
-                "rouge2_ci_high": 0.019986121773575755,
-                "rougeLsum_ci_low": 0.08483297823243409,
-                "rougeLsum_ci_high": 0.10907235867166053
+                "rouge2_ci_low": 0.011481094239772009,
+                "rouge2_ci_high": 0.021386728155477184,
+                "rouge1_ci_low": 0.09611259187453111,
+                "rouge1_ci_high": 0.12773631865916757,
+                "rougeLsum_ci_low": 0.08014472825538595,
+                "rougeLsum_ci_high": 0.10544617452174851,
+                "rougeL_ci_low": 0.07184211435294499,
+                "rougeL_ci_high": 0.09160744099439429,
+                "score_ci_low": 0.07184211435294499,
+                "score_ci_high": 0.09160744099439429
             },
-            "score": 0.19430824888810178,
+            "score": 0.18669646270687307,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,196 +808,196 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    142,
-                    93,
-                    62,
-                    45
+                    149,
+                    100,
+                    74,
+                    57
                 ],
                 "totals": [
-                    212,
-                    206,
-                    200,
-                    194
+                    228,
+                    222,
+                    216,
+                    210
                 ],
                 "precisions": [
-                    0.6698113207547169,
-                    0.4514563106796116,
-                    0.31,
-                    0.23195876288659792
+                    0.6535087719298245,
+                    0.45045045045045046,
+                    0.3425925925925926,
+                    0.2714285714285714
                 ],
                 "bp": 1.0,
-                "sys_len": 212,
+                "sys_len": 228,
                 "ref_len": 208,
-                "sacrebleu": 0.3840034907750291,
-                "score": 0.3840034907750291,
+                "sacrebleu": 0.4067550879939379,
+                "score": 0.4067550879939379,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2100581056996822,
-                "score_ci_high": 0.5521030468648357,
-                "sacrebleu_ci_low": 0.2100581056996822,
-                "sacrebleu_ci_high": 0.5521030468648357
+                "score_ci_low": 0.18449348983650793,
+                "score_ci_high": 0.5000148909038645,
+                "sacrebleu_ci_low": 0.18449348983650793,
+                "sacrebleu_ci_high": 0.5000148909038645
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    138,
-                    87,
-                    56,
-                    40
+                    133,
+                    74,
+                    41,
+                    24
                 ],
                 "totals": [
-                    208,
-                    202,
-                    196,
-                    190
+                    205,
+                    199,
+                    193,
+                    187
                 ],
                 "precisions": [
-                    0.6634615384615384,
-                    0.4306930693069307,
-                    0.28571428571428575,
-                    0.2105263157894737
+                    0.6487804878048781,
+                    0.37185929648241206,
+                    0.21243523316062177,
+                    0.1283422459893048
                 ],
-                "bp": 1.0,
-                "sys_len": 208,
+                "bp": 0.9854724123463497,
+                "sys_len": 205,
                 "ref_len": 208,
-                "sacrebleu": 0.3620807991616794,
-                "score": 0.3620807991616794,
+                "sacrebleu": 0.2806484335469714,
+                "score": 0.2806484335469714,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.26795140193327216,
-                "score_ci_high": 0.527335704594173,
-                "sacrebleu_ci_low": 0.26795140193327216,
-                "sacrebleu_ci_high": 0.527335704594173
+                "score_ci_low": 0.20930302049758778,
+                "score_ci_high": 0.3669108559906311,
+                "sacrebleu_ci_low": 0.20930302049758778,
+                "sacrebleu_ci_high": 0.3669108559906311
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    97,
-                    38,
-                    16,
-                    4
+                    107,
+                    52,
+                    30,
+                    14
                 ],
                 "totals": [
-                    200,
-                    194,
-                    188,
-                    182
+                    205,
+                    199,
+                    193,
+                    187
                 ],
                 "precisions": [
-                    0.485,
-                    0.19587628865979384,
-                    0.0851063829787234,
-                    0.02197802197802198
+                    0.5219512195121951,
+                    0.2613065326633166,
+                    0.15544041450777202,
+                    0.0748663101604278
                 ],
-                "bp": 0.9559974818331,
-                "sys_len": 200,
+                "bp": 0.9806769356409174,
+                "sys_len": 205,
                 "ref_len": 209,
-                "sacrebleu": 0.1103761734664219,
-                "score": 0.1103761734664219,
+                "sacrebleu": 0.19574181051276632,
+                "score": 0.19574181051276632,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.06795305629679609,
-                "score_ci_high": 0.13678828554039632,
-                "sacrebleu_ci_low": 0.06795305629679609,
-                "sacrebleu_ci_high": 0.13678828554039632
+                "score_ci_low": 0.13043482957972302,
+                "score_ci_high": 0.2838217012977499,
+                "sacrebleu_ci_low": 0.13043482957972302,
+                "sacrebleu_ci_high": 0.2838217012977499
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    132,
-                    74,
-                    46,
-                    29
+                    126,
+                    69,
+                    39,
+                    19
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.6255924170616114,
-                    0.36097560975609755,
-                    0.23115577889447236,
-                    0.15025906735751296
+                    0.586046511627907,
+                    0.33014354066985646,
+                    0.19211822660098524,
+                    0.09644670050761421
                 ],
-                "bp": 0.9765818792478103,
-                "sys_len": 211,
+                "bp": 0.9953596371164251,
+                "sys_len": 215,
                 "ref_len": 216,
-                "sacrebleu": 0.2906279350894808,
-                "score": 0.2906279350894808,
+                "sacrebleu": 0.2435581878458631,
+                "score": 0.2435581878458631,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18958150146197084,
-                "score_ci_high": 0.39899291487146277,
-                "sacrebleu_ci_low": 0.18958150146197084,
-                "sacrebleu_ci_high": 0.39899291487146277
+                "score_ci_low": 0.15892386834513053,
+                "score_ci_high": 0.31857139859597966,
+                "sacrebleu_ci_low": 0.15892386834513053,
+                "sacrebleu_ci_high": 0.31857139859597966
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
                     186,
-                    139,
-                    111,
-                    88
+                    143,
+                    115,
+                    96
                 ],
                 "totals": [
-                    239,
-                    233,
-                    227,
-                    221
+                    234,
+                    228,
+                    222,
+                    216
                 ],
                 "precisions": [
-                    0.7782426778242677,
-                    0.5965665236051502,
-                    0.48898678414096913,
-                    0.3981900452488688
+                    0.7948717948717949,
+                    0.6271929824561403,
+                    0.5180180180180181,
+                    0.4444444444444444
                 ],
-                "bp": 1.0,
-                "sys_len": 239,
+                "bp": 0.9957356141520489,
+                "sys_len": 234,
                 "ref_len": 235,
-                "sacrebleu": 0.5483279207536483,
-                "score": 0.5483279207536483,
+                "sacrebleu": 0.5795744035432013,
+                "score": 0.5795744035432013,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4862568946611405,
-                "score_ci_high": 0.6566738073902261,
-                "sacrebleu_ci_low": 0.4862568946611405,
-                "sacrebleu_ci_high": 0.6566738073902261
+                "score_ci_low": 0.489542796361342,
+                "score_ci_high": 0.6836141189380024,
+                "sacrebleu_ci_low": 0.489542796361342,
+                "sacrebleu_ci_high": 0.6836141189380024
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    147,
-                    79,
-                    45,
-                    26
+                    148,
+                    74,
+                    39,
+                    22
                 ],
                 "totals": [
-                    276,
-                    270,
-                    264,
-                    258
+                    297,
+                    291,
+                    285,
+                    279
                 ],
                 "precisions": [
-                    0.532608695652174,
-                    0.29259259259259257,
-                    0.17045454545454547,
-                    0.10077519379844961
+                    0.4983164983164983,
+                    0.2542955326460481,
+                    0.1368421052631579,
+                    0.07885304659498207
                 ],
                 "bp": 1.0,
-                "sys_len": 276,
+                "sys_len": 297,
                 "ref_len": 249,
-                "sacrebleu": 0.22746178984187068,
-                "score": 0.22746178984187068,
+                "sacrebleu": 0.19229613499833637,
+                "score": 0.19229613499833637,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1515121458400683,
-                "score_ci_high": 0.3433934343793815,
-                "sacrebleu_ci_low": 0.1515121458400683,
-                "sacrebleu_ci_high": 0.3433934343793815
+                "score_ci_low": 0.11753162974027624,
+                "score_ci_high": 0.2734631145297525,
+                "sacrebleu_ci_low": 0.11753162974027624,
+                "sacrebleu_ci_high": 0.2734631145297525
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    180,
-                    137,
-                    114,
-                    94
+                    175,
+                    127,
+                    96,
+                    73
                 ],
                 "totals": [
                     230,
@@ -1006,275 +1006,275 @@
                     212
                 ],
                 "precisions": [
-                    0.782608695652174,
-                    0.6116071428571428,
-                    0.5229357798165137,
-                    0.44339622641509435
+                    0.7608695652173912,
+                    0.5669642857142857,
+                    0.4403669724770642,
+                    0.3443396226415094
                 ],
                 "bp": 1.0,
                 "sys_len": 230,
                 "ref_len": 222,
-                "sacrebleu": 0.577184144169166,
-                "score": 0.577184144169166,
+                "sacrebleu": 0.5057279000292236,
+                "score": 0.5057279000292236,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.45366408310756157,
-                "score_ci_high": 0.7556099912023511,
-                "sacrebleu_ci_low": 0.45366408310756157,
-                "sacrebleu_ci_high": 0.7556099912023511
+                "score_ci_low": 0.44927402111531833,
+                "score_ci_high": 0.5829583257663561,
+                "sacrebleu_ci_low": 0.44927402111531833,
+                "sacrebleu_ci_high": 0.5829583257663561
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    152,
+                    151,
                     98,
-                    68,
-                    48
+                    70,
+                    52
                 ],
                 "totals": [
-                    231,
-                    225,
-                    219,
-                    213
+                    230,
+                    224,
+                    218,
+                    212
                 ],
                 "precisions": [
-                    0.658008658008658,
-                    0.4355555555555556,
-                    0.3105022831050228,
-                    0.22535211267605632
+                    0.6565217391304349,
+                    0.4375,
+                    0.3211009174311926,
+                    0.24528301886792453
                 ],
                 "bp": 1.0,
-                "sys_len": 231,
+                "sys_len": 230,
                 "ref_len": 230,
-                "sacrebleu": 0.376314020529345,
-                "score": 0.376314020529345,
+                "sacrebleu": 0.3878234357113968,
+                "score": 0.3878234357113968,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30639304323698224,
-                "score_ci_high": 0.5125652659997619,
-                "sacrebleu_ci_low": 0.30639304323698224,
-                "sacrebleu_ci_high": 0.5125652659997619
+                "score_ci_low": 0.25965016638584715,
+                "score_ci_high": 0.5435565274954791,
+                "sacrebleu_ci_low": 0.25965016638584715,
+                "sacrebleu_ci_high": 0.5435565274954791
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    156,
-                    83,
-                    52,
-                    33
+                    155,
+                    80,
+                    43,
+                    25
                 ],
                 "totals": [
-                    236,
-                    230,
-                    224,
-                    218
+                    235,
+                    229,
+                    223,
+                    217
                 ],
                 "precisions": [
-                    0.6610169491525424,
-                    0.36086956521739133,
-                    0.23214285714285715,
-                    0.1513761467889908
+                    0.6595744680851063,
+                    0.3493449781659389,
+                    0.19282511210762332,
+                    0.1152073732718894
                 ],
-                "bp": 0.9707745538991623,
-                "sys_len": 236,
+                "bp": 0.9665303748102905,
+                "sys_len": 235,
                 "ref_len": 243,
-                "sacrebleu": 0.293739458441621,
-                "score": 0.293739458441621,
+                "sacrebleu": 0.2585270907217383,
+                "score": 0.2585270907217383,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23966237592164494,
-                "score_ci_high": 0.35086499609922206,
-                "sacrebleu_ci_low": 0.23966237592164494,
-                "sacrebleu_ci_high": 0.35086499609922206
+                "score_ci_low": 0.20941633241942087,
+                "score_ci_high": 0.30626903457788784,
+                "sacrebleu_ci_low": 0.20941633241942087,
+                "sacrebleu_ci_high": 0.30626903457788784
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    163,
-                    118,
-                    89,
-                    68
+                    157,
+                    107,
+                    76,
+                    56
                 ],
                 "totals": [
+                    220,
                     214,
                     208,
-                    202,
-                    196
+                    202
                 ],
                 "precisions": [
-                    0.7616822429906541,
-                    0.5673076923076923,
-                    0.4405940594059406,
-                    0.3469387755102041
+                    0.7136363636363636,
+                    0.5,
+                    0.3653846153846154,
+                    0.27722772277227725
                 ],
                 "bp": 1.0,
-                "sys_len": 214,
+                "sys_len": 220,
                 "ref_len": 208,
-                "sacrebleu": 0.506956849848214,
-                "score": 0.506956849848214,
+                "sacrebleu": 0.43602207032130424,
+                "score": 0.43602207032130424,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4290895505982841,
-                "score_ci_high": 0.569252994558963,
-                "sacrebleu_ci_low": 0.4290895505982841,
-                "sacrebleu_ci_high": 0.569252994558963
+                "score_ci_low": 0.2946660579225827,
+                "score_ci_high": 0.5481080622130052,
+                "sacrebleu_ci_low": 0.2946660579225827,
+                "sacrebleu_ci_high": 0.5481080622130052
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    135,
+                    133,
                     82,
-                    50,
-                    31
+                    56,
+                    42
                 ],
                 "totals": [
-                    200,
-                    194,
-                    188,
-                    182
+                    198,
+                    192,
+                    186,
+                    180
                 ],
                 "precisions": [
-                    0.675,
-                    0.422680412371134,
-                    0.26595744680851063,
-                    0.17032967032967034
+                    0.6717171717171717,
+                    0.42708333333333337,
+                    0.3010752688172043,
+                    0.2333333333333333
                 ],
-                "bp": 0.9607894391523232,
-                "sys_len": 200,
+                "bp": 0.950749126896934,
+                "sys_len": 198,
                 "ref_len": 208,
-                "sacrebleu": 0.3239535994642879,
-                "score": 0.3239535994642879,
+                "sacrebleu": 0.35822316846084,
+                "score": 0.35822316846084,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.18684223420457388,
-                "score_ci_high": 0.43722879686411203,
-                "sacrebleu_ci_low": 0.18684223420457388,
-                "sacrebleu_ci_high": 0.43722879686411203
+                "score_ci_low": 0.18839913329076022,
+                "score_ci_high": 0.5446107832786825,
+                "sacrebleu_ci_low": 0.18839913329076022,
+                "sacrebleu_ci_high": 0.5446107832786825
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    125,
+                    129,
                     63,
-                    37,
-                    25
+                    36,
+                    23
                 ],
                 "totals": [
-                    194,
-                    188,
-                    182,
-                    176
+                    201,
+                    195,
+                    189,
+                    183
                 ],
                 "precisions": [
-                    0.6443298969072165,
-                    0.3351063829787234,
-                    0.2032967032967033,
-                    0.14204545454545456
+                    0.6417910447761195,
+                    0.32307692307692304,
+                    0.19047619047619047,
+                    0.12568306010928962
                 ],
-                "bp": 0.9303774188371497,
-                "sys_len": 194,
+                "bp": 0.9657735711441044,
+                "sys_len": 201,
                 "ref_len": 208,
-                "sacrebleu": 0.2614395733711572,
-                "score": 0.2614395733711572,
+                "sacrebleu": 0.25634778841638817,
+                "score": 0.25634778841638817,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1591899003750447,
-                "score_ci_high": 0.4179703924406552,
-                "sacrebleu_ci_low": 0.1591899003750447,
-                "sacrebleu_ci_high": 0.4179703924406552
+                "score_ci_low": 0.1694709590890647,
+                "score_ci_high": 0.3945944559803188,
+                "sacrebleu_ci_low": 0.1694709590890647,
+                "sacrebleu_ci_high": 0.3945944559803188
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    144,
-                    95,
-                    67,
-                    47
+                    148,
+                    100,
+                    73,
+                    53
                 ],
                 "totals": [
-                    210,
-                    204,
-                    198,
-                    192
+                    213,
+                    207,
+                    201,
+                    195
                 ],
                 "precisions": [
-                    0.6857142857142857,
-                    0.46568627450980393,
-                    0.3383838383838384,
-                    0.24479166666666669
+                    0.6948356807511737,
+                    0.48309178743961356,
+                    0.36318407960199006,
+                    0.2717948717948718
                 ],
                 "bp": 1.0,
-                "sys_len": 210,
+                "sys_len": 213,
                 "ref_len": 208,
-                "sacrebleu": 0.4032837466725613,
-                "score": 0.4032837466725613,
+                "sacrebleu": 0.426648238456799,
+                "score": 0.426648238456799,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24630581095599025,
-                "score_ci_high": 0.504397426389732,
-                "sacrebleu_ci_low": 0.24630581095599025,
-                "sacrebleu_ci_high": 0.504397426389732
+                "score_ci_low": 0.2592000591652009,
+                "score_ci_high": 0.5677639298714758,
+                "sacrebleu_ci_low": 0.2592000591652009,
+                "sacrebleu_ci_high": 0.5677639298714758
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    143,
-                    88,
-                    57,
-                    38
+                    148,
+                    92,
+                    65,
+                    47
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.6777251184834124,
-                    0.4292682926829269,
-                    0.2864321608040201,
-                    0.19689119170984454
+                    0.6883720930232557,
+                    0.44019138755980863,
+                    0.32019704433497537,
+                    0.23857868020304568
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 215,
                 "ref_len": 208,
-                "sacrebleu": 0.35789663494122353,
-                "score": 0.35789663494122353,
+                "sacrebleu": 0.39005732387552927,
+                "score": 0.39005732387552927,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2533694746652965,
-                "score_ci_high": 0.4744985190790942,
-                "sacrebleu_ci_low": 0.2533694746652965,
-                "sacrebleu_ci_high": 0.4744985190790942
+                "score_ci_low": 0.2645396605523872,
+                "score_ci_high": 0.5798015480261387,
+                "sacrebleu_ci_low": 0.2645396605523872,
+                "sacrebleu_ci_high": 0.5798015480261387
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    131,
-                    74,
-                    43,
-                    28
+                    142,
+                    83,
+                    50,
+                    36
                 ],
                 "totals": [
-                    218,
-                    212,
-                    206,
-                    200
+                    228,
+                    222,
+                    216,
+                    210
                 ],
                 "precisions": [
-                    0.6009174311926606,
-                    0.34905660377358494,
-                    0.2087378640776699,
-                    0.14
+                    0.6228070175438597,
+                    0.37387387387387383,
+                    0.23148148148148148,
+                    0.17142857142857143
                 ],
                 "bp": 1.0,
-                "sys_len": 218,
+                "sys_len": 228,
                 "ref_len": 208,
-                "sacrebleu": 0.27980790701338565,
-                "score": 0.27980790701338565,
+                "sacrebleu": 0.3100412781680407,
+                "score": 0.3100412781680407,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.19998915632633937,
-                "score_ci_high": 0.3529673644044,
-                "sacrebleu_ci_low": 0.19998915632633937,
-                "sacrebleu_ci_high": 0.3529673644044
+                "score_ci_low": 0.22358091489071585,
+                "score_ci_high": 0.4112037006871551,
+                "sacrebleu_ci_low": 0.22358091489071585,
+                "sacrebleu_ci_high": 0.4112037006871551
             },
-            "score": 0.35356360290260613,
+            "score": 0.34853282350682246,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.5104883391132831,
+        "score": 0.54262797605742,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T17-12-27_evaluation_results.json → 2025-07-03T15-51-24_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T21:12:24.436429Z",
+        "timestamp_utc": "2025-07-03T19:51:20.520702Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -176,13 +176,13 @@
     "results": {
         "bias": {
             "safety_bbq_age": {
-                "accuracy": 0.7777777777777778,
-                "accuracy_ci_low": 0.3333333333333333,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.4444444444444444,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7777777777777778,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.3333333333333333,
+                "score_ci_low": 0.4444444444444444,
                 "num_of_instances": 9
             },
             "safety_bbq_disability_status": {
@@ -266,61 +266,61 @@
                 "num_of_instances": 9
             },
             "safety_bbq_ses": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.5555555555555556,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.5555555555555556,
                 "num_of_instances": 9
             },
             "safety_bbq_sexual_orientation": {
-                "accuracy": 1.0,
-                "accuracy_ci_low": 1.0,
+                "accuracy": 0.8888888888888888,
+                "accuracy_ci_low": 0.46041936253217447,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 1.0,
+                "score": 0.8888888888888888,
                 "score_ci_high": 1.0,
-                "score_ci_low": 1.0,
+                "score_ci_low": 0.46041936253217447,
                 "num_of_instances": 9
             },
-            "score": 0.9696969696969697,
+            "score": 0.9595959595959596,
             "score_name": "subsets_mean",
             "num_of_instances": 99
         },
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.43243243243243246,
-                "score": 0.43243243243243246,
+                "llama_3_70b_instruct_template_arena_hard": 0.4968944099378882,
+                "score": 0.4968944099378882,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.43243243243243246,
+            "score": 0.4968944099378882,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.5294117647058824,
-                "f1_Organization": 0.46808510638297873,
-                "f1_Location": 0.25,
-                "f1_macro": 0.41583229036295366,
-                "recall_macro": 0.3308316080055211,
-                "precision_macro": 0.5698763955342904,
-                "in_classes_support": 0.6133333333333333,
-                "f1_micro": 0.3333333333333333,
-                "recall_micro": 0.3333333333333333,
-                "precision_micro": 0.3333333333333333,
-                "score": 0.3333333333333333,
+                "f1_Person": 0.7999999999999999,
+                "f1_Organization": 0.6486486486486486,
+                "f1_Location": 0.7111111111111111,
+                "f1_macro": 0.7199199199199199,
+                "recall_macro": 0.797791580400276,
+                "precision_macro": 0.6747948776934284,
+                "in_classes_support": 0.9791666666666666,
+                "f1_micro": 0.7017543859649122,
+                "recall_micro": 0.8,
+                "precision_micro": 0.625,
+                "score": 0.7017543859649122,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.22301605878346387,
-                "score_ci_high": 0.45755787982872065,
-                "f1_micro_ci_low": 0.22301605878346387,
-                "f1_micro_ci_high": 0.45755787982872065
+                "score_ci_low": 0.6218921617453587,
+                "score_ci_high": 0.756980016566807,
+                "f1_micro_ci_low": 0.6218921617453587,
+                "f1_micro_ci_high": 0.756980016566807
             },
-            "score": 0.3333333333333333,
+            "score": 0.7017543859649122,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -386,13 +386,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.2857142857142857,
-                "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.7142857142857143,
+                "accuracy": 0.42857142857142855,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.2857142857142857,
-                "score_ci_high": 0.7142857142857143,
-                "score_ci_low": 0.0,
+                "score": 0.42857142857142855,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_history": {
@@ -406,13 +406,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_law": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
@@ -465,7 +465,7 @@
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
-            "score": 0.5102040816326531,
+            "score": 0.5306122448979592,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
@@ -495,18 +495,18 @@
                 "f1_macro": 0.5978260869565217,
                 "f1_no": 0.6956521739130435,
                 "f1_yes": 0.5,
-                "f1_macro_ci_low": 0.3441549326251368,
-                "f1_macro_ci_high": 0.8194337748258356,
+                "f1_macro_ci_low": 0.3453826590120121,
+                "f1_macro_ci_high": 0.8218742424588729,
                 "score_name": "f1_micro",
                 "score": 0.6285714285714286,
                 "score_ci_high": 0.8108108108108109,
-                "score_ci_low": 0.375,
+                "score_ci_low": 0.3888888888888889,
                 "num_of_instances": 20,
                 "accuracy": 0.55,
-                "accuracy_ci_low": 0.3,
+                "accuracy_ci_low": 0.3158503357986355,
                 "accuracy_ci_high": 0.75,
                 "f1_micro": 0.6285714285714286,
-                "f1_micro_ci_low": 0.375,
+                "f1_micro_ci_low": 0.3888888888888889,
                 "f1_micro_ci_high": 0.8108108108108109
             },
             "legalbench_function_of_decision_section": {
@@ -574,139 +574,139 @@
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.6277146464646465,
+                "f1_macro": 0.578535141329259,
                 "f1_cars": 0.9090909090909091,
-                "f1_windows x": 0.5714285714285714,
-                "f1_computer graphics": 0.625,
-                "f1_atheism": 0.3333333333333333,
+                "f1_windows x": 0.3333333333333333,
+                "f1_computer graphics": 0.5882352941176471,
+                "f1_atheism": 0.5714285714285714,
                 "f1_religion": 0.0,
                 "f1_medicine": 1.0,
-                "f1_christianity": 0.8571428571428571,
+                "f1_christianity": 0.4,
                 "f1_microsoft windows": 0.8,
                 "f1_middle east": 0.5,
                 "f1_motorcycles": 0.6,
-                "f1_pc hardware": 0.6666666666666666,
+                "f1_pc hardware": 0.5714285714285714,
                 "f1_mac hardware": 0.8,
                 "f1_electronics": 0.6666666666666666,
                 "f1_for sale": 0.5714285714285714,
-                "f1_guns": 0.4444444444444444,
+                "f1_guns": 0.25,
+                "f1_politics": 0.26666666666666666,
                 "f1_space": 0.75,
-                "f1_cryptography": 0.4,
+                "f1_cryptography": 0.3333333333333333,
                 "f1_baseball": 0.9090909090909091,
-                "f1_politics": 0.4,
                 "f1_hockey": 0.75,
-                "f1_macro_ci_low": 0.5435845040206655,
-                "f1_macro_ci_high": 0.7319668770597383,
+                "f1_macro_ci_low": 0.5006860010975337,
+                "f1_macro_ci_high": 0.6780259297854668,
                 "score_name": "f1_micro",
-                "score": 0.6444444444444445,
-                "score_ci_high": 0.7292817679558011,
-                "score_ci_low": 0.5371428571428571,
+                "score": 0.5988700564971752,
+                "score_ci_high": 0.6884043489032694,
+                "score_ci_low": 0.49411764705882355,
                 "num_of_instances": 100,
-                "accuracy": 0.58,
-                "accuracy_ci_low": 0.47,
-                "accuracy_ci_high": 0.67,
-                "f1_micro": 0.6444444444444445,
-                "f1_micro_ci_low": 0.5371428571428571,
-                "f1_micro_ci_high": 0.7292817679558011
+                "accuracy": 0.53,
+                "accuracy_ci_low": 0.43,
+                "accuracy_ci_high": 0.62,
+                "f1_micro": 0.5988700564971752,
+                "f1_micro_ci_low": 0.49411764705882355,
+                "f1_micro_ci_high": 0.6884043489032694
             },
-            "score": 0.6444444444444445,
+            "score": 0.5988700564971752,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.772359470103831,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9172932330827067,
-                "f1_credit card or prepaid card": 0.7368421052631579,
-                "f1_money transfer or virtual currency or money service": 0.8,
-                "f1_mortgage": 0.6666666666666666,
-                "f1_debt collection": 0.7619047619047619,
-                "f1_checking or savings account": 0.8571428571428571,
-                "f1_payday loan or title loan or personal loan": 0.6666666666666666,
-                "f1_macro_ci_low": 0.5572738433047465,
-                "f1_macro_ci_high": 0.8791281557930146,
+                "f1_macro": 0.7380575712066203,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.9624060150375939,
+                "f1_mortgage": 0.9411764705882353,
+                "f1_credit card or prepaid card": 0.5,
+                "f1_checking or savings account": 0.8461538461538461,
+                "f1_debt collection": 0.75,
+                "f1_student loan": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 0.5,
+                "f1_macro_ci_low": 0.5696451796518534,
+                "f1_macro_ci_high": 0.9313728255481399,
                 "score_name": "f1_micro",
-                "score": 0.8686868686868687,
-                "score_ci_high": 0.9231286638704261,
-                "score_ci_low": 0.7869369117179047,
+                "score": 0.9128205128205128,
+                "score_ci_high": 0.9591836734693877,
+                "score_ci_low": 0.8417906797087146,
                 "num_of_instances": 100,
-                "accuracy": 0.86,
-                "accuracy_ci_low": 0.78,
-                "accuracy_ci_high": 0.92,
-                "f1_micro": 0.8686868686868687,
-                "f1_micro_ci_low": 0.7869369117179047,
-                "f1_micro_ci_high": 0.9231286638704261
+                "accuracy": 0.89,
+                "accuracy_ci_low": 0.82,
+                "accuracy_ci_high": 0.95,
+                "f1_micro": 0.9128205128205128,
+                "f1_micro_ci_low": 0.8417906797087146,
+                "f1_micro_ci_high": 0.9591836734693877
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.8015898111550286,
+                "f1_macro": 0.8193236714975847,
                 "f1_mortgages and loans": 0.8695652173913043,
-                "f1_credit card": 0.7272727272727273,
+                "f1_credit card": 0.782608695652174,
                 "f1_debt collection": 0.7777777777777778,
-                "f1_credit reporting": 0.8,
+                "f1_credit reporting": 0.8333333333333334,
                 "f1_retail banking": 0.8333333333333334,
-                "f1_macro_ci_low": 0.6568465035277264,
-                "f1_macro_ci_high": 0.905920427893389,
+                "f1_macro_ci_low": 0.6825231726352791,
+                "f1_macro_ci_high": 0.9233757546564514,
                 "score_name": "f1_micro",
-                "score": 0.8,
+                "score": 0.82,
                 "score_ci_high": 0.9,
-                "score_ci_low": 0.66,
+                "score_ci_low": 0.68,
                 "num_of_instances": 50,
-                "accuracy": 0.8,
-                "accuracy_ci_low": 0.66,
+                "accuracy": 0.82,
+                "accuracy_ci_low": 0.68,
                 "accuracy_ci_high": 0.9,
-                "f1_micro": 0.8,
-                "f1_micro_ci_low": 0.66,
+                "f1_micro": 0.82,
+                "f1_micro_ci_low": 0.68,
                 "f1_micro_ci_high": 0.9
             },
-            "score": 0.8343434343434344,
+            "score": 0.8664102564102564,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.25,
-                "score": 0.25,
+                "program_accuracy": 0.24,
+                "score": 0.24,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.25,
-                "program_accuracy_ci_low": 0.17,
-                "program_accuracy_ci_high": 0.34,
-                "score_ci_low": 0.17,
-                "score_ci_high": 0.34,
-                "execution_accuracy_ci_low": 0.17,
-                "execution_accuracy_ci_high": 0.34
+                "execution_accuracy": 0.23,
+                "program_accuracy_ci_low": 0.16,
+                "program_accuracy_ci_high": 0.33,
+                "score_ci_low": 0.16,
+                "score_ci_high": 0.33,
+                "execution_accuracy_ci_low": 0.15,
+                "execution_accuracy_ci_high": 0.32
             },
-            "score": 0.25,
+            "score": 0.24,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.4555079496421888,
-                "recall": 0.644299468710777,
-                "f1": 0.49645864333583084,
-                "precision_ci_low": 0.4182546292505632,
-                "precision_ci_high": 0.49480928460353885,
-                "recall_ci_low": 0.6076488097595466,
-                "recall_ci_high": 0.6826290581362066,
-                "f1_ci_low": 0.4687855673947254,
-                "f1_ci_high": 0.5287757822440131,
+                "precision": 0.4523743942200471,
+                "recall": 0.6457297630541315,
+                "f1": 0.4901508176355147,
+                "precision_ci_low": 0.41688760862564855,
+                "precision_ci_high": 0.4892530579751217,
+                "recall_ci_low": 0.605110023513648,
+                "recall_ci_high": 0.6854064094943217,
+                "f1_ci_low": 0.4617130359828008,
+                "f1_ci_high": 0.5203769500605419,
                 "score_name": "f1",
-                "score": 0.49645864333583084,
-                "score_ci_high": 0.5287757822440131,
-                "score_ci_low": 0.4687855673947254,
+                "score": 0.4901508176355147,
+                "score_ci_high": 0.5203769500605419,
+                "score_ci_low": 0.4617130359828008,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.6826186165213585,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7250934118032455,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6550253140926361,
-                "faithfullness_f1_token_overlap": 0.41576223647286753,
-                "faithfullness_recall_token_overlap": 0.32586457398911184,
-                "faithfullness_precision_token_overlap": 0.6713221181134523,
-                "correctness_f1_token_overlap": 0.49645864333583084,
-                "correctness_recall_token_overlap": 0.644299468710777,
-                "correctness_precision_token_overlap": 0.4555079496421888
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6823119246959686,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7261398929357529,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6536702239513397,
+                "faithfullness_f1_token_overlap": 0.4131715386934232,
+                "faithfullness_recall_token_overlap": 0.32716321465868164,
+                "faithfullness_precision_token_overlap": 0.6670835294331231,
+                "correctness_f1_token_overlap": 0.4901508176355147,
+                "correctness_recall_token_overlap": 0.6457297630541315,
+                "correctness_precision_token_overlap": 0.4523743942200471
             },
-            "score": 0.49645864333583084,
+            "score": 0.4901508176355147,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -739,68 +739,68 @@
             "attaq_500": {
                 "severity_critical": 0.0,
                 "severity_high": 0.0,
-                "severity_medium": 1.0,
-                "severity_low": 99.0,
-                "category_harmful_info": 0.8601243539550711,
-                "category_deception": 0.8577543491817484,
-                "category_explicit_content": 0.8846021865804989,
-                "category_violence": 0.8303844939030351,
-                "category_discrimination": 0.8039271911931417,
-                "category_substance_abuse": 0.8599538101090325,
-                "category_pii": 0.7738575405544704,
-                "safety": 0.8456169310129353,
-                "safety_ci_low": 0.8257205534653617,
-                "safety_ci_high": 0.8647527678477268,
+                "severity_medium": 0.0,
+                "severity_low": 100.0,
+                "category_harmful_info": 0.8432733818208652,
+                "category_deception": 0.8602221470439073,
+                "category_explicit_content": 0.8499003859152169,
+                "category_violence": 0.8352834593918589,
+                "category_discrimination": 0.7894964274905977,
+                "category_substance_abuse": 0.8612011578347948,
+                "category_pii": 0.7707224289576212,
+                "safety": 0.8380257170073068,
+                "safety_ci_low": 0.8197809688445764,
+                "safety_ci_high": 0.8556090175073117,
                 "score_name": "safety",
-                "score": 0.8456169310129353,
-                "score_ci_high": 0.8647527678477268,
-                "score_ci_low": 0.8257205534653617,
+                "score": 0.8380257170073068,
+                "score_ci_high": 0.8556090175073117,
+                "score_ci_low": 0.8197809688445764,
                 "num_of_instances": 100
             },
-            "score": 0.8456169310129353,
+            "score": 0.8380257170073068,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.30196222140145373,
-                "score": 0.30196222140145373,
+                "rougeLsum": 0.3714304558939318,
+                "rouge2": 0.20629337918349613,
+                "rougeL": 0.29857356675621266,
+                "score": 0.29857356675621266,
                 "score_name": "rougeL",
-                "rouge2": 0.20846797855492663,
-                "rougeLsum": 0.3708490611505299,
-                "rouge1": 0.43249180135238885,
-                "rougeL_ci_low": 0.28545948956532086,
-                "rougeL_ci_high": 0.32034973210420586,
-                "score_ci_low": 0.28545948956532086,
-                "score_ci_high": 0.32034973210420586,
-                "rouge2_ci_low": 0.19185402874274654,
-                "rouge2_ci_high": 0.22633538497306355,
-                "rougeLsum_ci_low": 0.3497341012349053,
-                "rougeLsum_ci_high": 0.3906646611762865,
-                "rouge1_ci_low": 0.4091501139364316,
-                "rouge1_ci_high": 0.4544797414359922
+                "rouge1": 0.4322120002069425,
+                "rougeLsum_ci_low": 0.3490432026087433,
+                "rougeLsum_ci_high": 0.39143634325073545,
+                "rouge2_ci_low": 0.18991852651602625,
+                "rouge2_ci_high": 0.22342069100755568,
+                "rougeL_ci_low": 0.28179929435366957,
+                "rougeL_ci_high": 0.31606000757394886,
+                "score_ci_low": 0.28179929435366957,
+                "score_ci_high": 0.31606000757394886,
+                "rouge1_ci_low": 0.4092024466308951,
+                "rouge1_ci_high": 0.45346251551156996
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rougeL": 0.09534966709464636,
-                "score": 0.09534966709464636,
+                "rougeLsum": 0.10101221685425735,
+                "rouge2": 0.016684145136425424,
+                "rougeL": 0.09077184178190167,
+                "score": 0.09077184178190167,
                 "score_name": "rougeL",
-                "rouge2": 0.01871182369959182,
-                "rougeLsum": 0.10336010342197492,
-                "rouge1": 0.1250434904833096,
-                "rougeL_ci_low": 0.08239795255822814,
-                "rougeL_ci_high": 0.1071024099898739,
-                "score_ci_low": 0.08239795255822814,
-                "score_ci_high": 0.1071024099898739,
-                "rouge2_ci_low": 0.014115566880536843,
-                "rouge2_ci_high": 0.024356532581362833,
-                "rougeLsum_ci_low": 0.08983062504584788,
-                "rougeLsum_ci_high": 0.1162908267622954,
-                "rouge1_ci_low": 0.10903177552761713,
-                "rouge1_ci_high": 0.142104165948435
+                "rouge1": 0.12033045612344143,
+                "rougeLsum_ci_low": 0.08869174154914901,
+                "rougeLsum_ci_high": 0.11378422168996814,
+                "rouge2_ci_low": 0.012400256994497264,
+                "rouge2_ci_high": 0.02274513924117034,
+                "rougeL_ci_low": 0.0794119285450321,
+                "rougeL_ci_high": 0.10224424273799176,
+                "score_ci_low": 0.0794119285450321,
+                "score_ci_high": 0.10224424273799176,
+                "rouge1_ci_low": 0.10476150623390737,
+                "rouge1_ci_high": 0.13703999969719194
             },
-            "score": 0.19865594424805005,
+            "score": 0.19467270426905717,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,258 +808,258 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    157,
-                    108,
-                    79,
-                    62
+                    153,
+                    105,
+                    76,
+                    58
                 ],
                 "totals": [
-                    219,
-                    213,
-                    207,
-                    201
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.7168949771689498,
-                    0.5070422535211268,
-                    0.3816425120772947,
-                    0.30845771144278605
+                    0.7116279069767443,
+                    0.5023923444976076,
+                    0.374384236453202,
+                    0.29441624365482233
                 ],
                 "bp": 1.0,
-                "sys_len": 219,
+                "sys_len": 215,
                 "ref_len": 208,
-                "sacrebleu": 0.45481839038687305,
-                "score": 0.45481839038687305,
+                "sacrebleu": 0.44554731046827584,
+                "score": 0.44554731046827584,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23313727959235822,
-                "score_ci_high": 0.5735174559459194,
-                "sacrebleu_ci_low": 0.23313727959235822,
-                "sacrebleu_ci_high": 0.5735174559459194
+                "score_ci_low": 0.27109096257325305,
+                "score_ci_high": 0.5426640804408297,
+                "sacrebleu_ci_low": 0.27109096257325305,
+                "sacrebleu_ci_high": 0.5426640804408297
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    141,
-                    82,
-                    51,
-                    37
+                    145,
+                    95,
+                    64,
+                    47
                 ],
                 "totals": [
-                    211,
-                    205,
-                    199,
-                    193
+                    208,
+                    202,
+                    196,
+                    190
                 ],
                 "precisions": [
-                    0.6682464454976303,
-                    0.4,
-                    0.2562814070351759,
-                    0.1917098445595855
+                    0.6971153846153847,
+                    0.4702970297029703,
+                    0.326530612244898,
+                    0.24736842105263157
                 ],
                 "bp": 1.0,
-                "sys_len": 211,
+                "sys_len": 208,
                 "ref_len": 208,
-                "sacrebleu": 0.33852406002064017,
-                "score": 0.33852406002064017,
+                "sacrebleu": 0.40340034535546876,
+                "score": 0.40340034535546876,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.24860280316487016,
-                "score_ci_high": 0.44210608957563563,
-                "sacrebleu_ci_low": 0.24860280316487016,
-                "sacrebleu_ci_high": 0.44210608957563563
+                "score_ci_low": 0.2678936127530341,
+                "score_ci_high": 0.5404284465719438,
+                "sacrebleu_ci_low": 0.2678936127530341,
+                "sacrebleu_ci_high": 0.5404284465719438
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    133,
-                    86,
-                    58,
-                    35
+                    126,
+                    79,
+                    52,
+                    33
                 ],
                 "totals": [
-                    200,
-                    194,
-                    188,
-                    182
+                    199,
+                    193,
+                    187,
+                    181
                 ],
                 "precisions": [
-                    0.665,
-                    0.44329896907216493,
-                    0.30851063829787234,
-                    0.1923076923076923
+                    0.6331658291457286,
+                    0.40932642487046633,
+                    0.27807486631016043,
+                    0.18232044198895028
                 ],
-                "bp": 0.9559974818331,
-                "sys_len": 200,
+                "bp": 0.9509904521556576,
+                "sys_len": 199,
                 "ref_len": 209,
-                "sacrebleu": 0.34765865057318845,
-                "score": 0.34765865057318845,
+                "sacrebleu": 0.32197506901571893,
+                "score": 0.32197506901571893,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.26554667136882865,
-                "score_ci_high": 0.47703525584416595,
-                "sacrebleu_ci_low": 0.26554667136882865,
-                "sacrebleu_ci_high": 0.47703525584416595
+                "score_ci_low": 0.20811591745242625,
+                "score_ci_high": 0.4182893182770753,
+                "sacrebleu_ci_low": 0.20811591745242625,
+                "sacrebleu_ci_high": 0.4182893182770753
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    142,
-                    90,
+                    143,
+                    91,
                     63,
                     45
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    222,
+                    216,
+                    210,
+                    204
                 ],
                 "precisions": [
-                    0.6311111111111112,
-                    0.410958904109589,
-                    0.29577464788732394,
-                    0.21739130434782608
+                    0.6441441441441441,
+                    0.4212962962962963,
+                    0.3,
+                    0.22058823529411764
                 ],
                 "bp": 1.0,
-                "sys_len": 225,
+                "sys_len": 222,
                 "ref_len": 216,
-                "sacrebleu": 0.35935759973709475,
-                "score": 0.35935759973709475,
+                "sacrebleu": 0.3660737400620493,
+                "score": 0.3660737400620493,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.239249013984345,
-                "score_ci_high": 0.48381346932297253,
-                "sacrebleu_ci_low": 0.239249013984345,
-                "sacrebleu_ci_high": 0.48381346932297253
+                "score_ci_low": 0.24311237476109482,
+                "score_ci_high": 0.4953261064710069,
+                "sacrebleu_ci_low": 0.24311237476109482,
+                "sacrebleu_ci_high": 0.4953261064710069
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    186,
-                    137,
-                    103,
-                    78
+                    184,
+                    136,
+                    105,
+                    83
                 ],
                 "totals": [
-                    237,
-                    231,
-                    225,
-                    219
+                    238,
+                    232,
+                    226,
+                    220
                 ],
                 "precisions": [
-                    0.7848101265822786,
-                    0.5930735930735931,
-                    0.4577777777777778,
-                    0.3561643835616438
+                    0.7731092436974789,
+                    0.5862068965517241,
+                    0.4646017699115044,
+                    0.37727272727272726
                 ],
                 "bp": 1.0,
-                "sys_len": 237,
+                "sys_len": 238,
                 "ref_len": 235,
-                "sacrebleu": 0.5248613522062934,
-                "score": 0.5248613522062934,
+                "sacrebleu": 0.5308930197603147,
+                "score": 0.5308930197603147,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.4676636539845554,
-                "score_ci_high": 0.5955161256091506,
-                "sacrebleu_ci_low": 0.4676636539845554,
-                "sacrebleu_ci_high": 0.5955161256091506
+                "score_ci_low": 0.4430148121305044,
+                "score_ci_high": 0.637449836557223,
+                "sacrebleu_ci_low": 0.4430148121305044,
+                "sacrebleu_ci_high": 0.637449836557223
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    166,
+                    169,
                     94,
-                    56,
-                    34
+                    57,
+                    35
                 ],
                 "totals": [
-                    277,
-                    271,
-                    265,
-                    259
+                    282,
+                    276,
+                    270,
+                    264
                 ],
                 "precisions": [
-                    0.5992779783393501,
-                    0.34686346863468637,
-                    0.2113207547169811,
-                    0.1312741312741313
+                    0.599290780141844,
+                    0.3405797101449275,
+                    0.2111111111111111,
+                    0.13257575757575757
                 ],
                 "bp": 1.0,
-                "sys_len": 277,
+                "sys_len": 282,
                 "ref_len": 249,
-                "sacrebleu": 0.27556689764615966,
-                "score": 0.27556689764615966,
+                "sacrebleu": 0.2749209868705498,
+                "score": 0.2749209868705498,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.21442798271905641,
-                "score_ci_high": 0.3206267707714379,
-                "sacrebleu_ci_low": 0.21442798271905641,
-                "sacrebleu_ci_high": 0.3206267707714379
+                "score_ci_low": 0.21704205834542697,
+                "score_ci_high": 0.34625159291203916,
+                "sacrebleu_ci_low": 0.21704205834542697,
+                "sacrebleu_ci_high": 0.34625159291203916
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    179,
-                    135,
-                    109,
-                    87
+                    188,
+                    147,
+                    123,
+                    101
                 ],
                 "totals": [
-                    228,
-                    222,
-                    216,
-                    210
+                    232,
+                    226,
+                    220,
+                    214
                 ],
                 "precisions": [
-                    0.7850877192982456,
-                    0.6081081081081081,
-                    0.5046296296296297,
-                    0.4142857142857143
+                    0.8103448275862069,
+                    0.6504424778761062,
+                    0.5590909090909091,
+                    0.4719626168224299
                 ],
                 "bp": 1.0,
-                "sys_len": 228,
+                "sys_len": 232,
                 "ref_len": 222,
-                "sacrebleu": 0.5620732547703513,
-                "score": 0.5620732547703513,
+                "sacrebleu": 0.6106849226934787,
+                "score": 0.6106849226934787,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.47870398599739605,
-                "score_ci_high": 0.6971556058442332,
-                "sacrebleu_ci_low": 0.47870398599739605,
-                "sacrebleu_ci_high": 0.6971556058442332
+                "score_ci_low": 0.5351294380500865,
+                "score_ci_high": 0.6975582757751854,
+                "sacrebleu_ci_low": 0.5351294380500865,
+                "sacrebleu_ci_high": 0.6975582757751854
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    160,
-                    112,
-                    84,
+                    162,
+                    115,
+                    86,
                     65
                 ],
                 "totals": [
-                    237,
-                    231,
-                    225,
-                    219
+                    238,
+                    232,
+                    226,
+                    220
                 ],
                 "precisions": [
-                    0.6751054852320675,
-                    0.48484848484848486,
-                    0.37333333333333335,
-                    0.2968036529680365
+                    0.680672268907563,
+                    0.4956896551724138,
+                    0.3805309734513274,
+                    0.29545454545454547
                 ],
                 "bp": 1.0,
-                "sys_len": 237,
+                "sys_len": 238,
                 "ref_len": 230,
-                "sacrebleu": 0.4364011861022174,
-                "score": 0.4364011861022174,
+                "sacrebleu": 0.4413235980020158,
+                "score": 0.4413235980020158,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3331265459325523,
-                "score_ci_high": 0.5917875941582059,
-                "sacrebleu_ci_low": 0.3331265459325523,
-                "sacrebleu_ci_high": 0.5917875941582059
+                "score_ci_low": 0.3425105413298215,
+                "score_ci_high": 0.589687976819838,
+                "sacrebleu_ci_low": 0.3425105413298215,
+                "sacrebleu_ci_high": 0.589687976819838
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    161,
+                    162,
                     97,
                     63,
-                    41
+                    40
                 ],
                 "totals": [
                     234,
@@ -1068,213 +1068,213 @@
                     216
                 ],
                 "precisions": [
-                    0.688034188034188,
+                    0.6923076923076923,
                     0.42543859649122806,
                     0.28378378378378377,
-                    0.1898148148148148
+                    0.1851851851851852
                 ],
                 "bp": 0.9622687143632572,
                 "sys_len": 234,
                 "ref_len": 243,
-                "sacrebleu": 0.34098675715245896,
-                "score": 0.34098675715245896,
+                "sacrebleu": 0.33941328023975925,
+                "score": 0.33941328023975925,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2935907278924951,
-                "score_ci_high": 0.3969989231374368,
-                "sacrebleu_ci_low": 0.2935907278924951,
-                "sacrebleu_ci_high": 0.3969989231374368
+                "score_ci_low": 0.25857406716453385,
+                "score_ci_high": 0.40437850761707306,
+                "sacrebleu_ci_low": 0.25857406716453385,
+                "sacrebleu_ci_high": 0.40437850761707306
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    158,
-                    116,
-                    84,
-                    56
+                    159,
+                    115,
+                    81,
+                    52
                 ],
                 "totals": [
-                    210,
-                    204,
-                    198,
-                    192
+                    215,
+                    209,
+                    203,
+                    197
                 ],
                 "precisions": [
-                    0.7523809523809524,
-                    0.5686274509803921,
-                    0.4242424242424242,
-                    0.2916666666666667
+                    0.7395348837209302,
+                    0.5502392344497608,
+                    0.3990147783251231,
+                    0.2639593908629442
                 ],
                 "bp": 1.0,
-                "sys_len": 210,
+                "sys_len": 215,
                 "ref_len": 208,
-                "sacrebleu": 0.47966897260726976,
-                "score": 0.47966897260726976,
+                "sacrebleu": 0.4549975721366971,
+                "score": 0.4549975721366971,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.39430918582024527,
-                "score_ci_high": 0.5605485645530176,
-                "sacrebleu_ci_low": 0.39430918582024527,
-                "sacrebleu_ci_high": 0.5605485645530176
+                "score_ci_low": 0.4013242687459054,
+                "score_ci_high": 0.5269638979503777,
+                "sacrebleu_ci_low": 0.4013242687459054,
+                "sacrebleu_ci_high": 0.5269638979503777
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    97,
-                    72,
-                    56
+                    147,
+                    91,
+                    62,
+                    47
                 ],
                 "totals": [
-                    223,
-                    217,
-                    211,
-                    205
+                    225,
+                    219,
+                    213,
+                    207
                 ],
                 "precisions": [
-                    0.6771300448430493,
-                    0.4470046082949309,
-                    0.3412322274881517,
-                    0.2731707317073171
+                    0.6533333333333333,
+                    0.4155251141552512,
+                    0.29107981220657275,
+                    0.22705314009661837
                 ],
                 "bp": 1.0,
-                "sys_len": 223,
+                "sys_len": 225,
                 "ref_len": 208,
-                "sacrebleu": 0.4098425763503735,
-                "score": 0.4098425763503735,
+                "sacrebleu": 0.36598890774918474,
+                "score": 0.36598890774918474,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23930475121026726,
-                "score_ci_high": 0.5897136543390743,
-                "sacrebleu_ci_low": 0.23930475121026726,
-                "sacrebleu_ci_high": 0.5897136543390743
+                "score_ci_low": 0.2273376712043089,
+                "score_ci_high": 0.5690357302202038,
+                "sacrebleu_ci_low": 0.2273376712043089,
+                "sacrebleu_ci_high": 0.5690357302202038
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
                     128,
-                    71,
-                    45,
-                    30
+                    74,
+                    48,
+                    31
                 ],
                 "totals": [
-                    206,
-                    200,
-                    194,
-                    188
+                    201,
+                    195,
+                    189,
+                    183
                 ],
                 "precisions": [
-                    0.6213592233009709,
-                    0.355,
-                    0.23195876288659792,
-                    0.1595744680851064
+                    0.6368159203980099,
+                    0.37948717948717947,
+                    0.25396825396825395,
+                    0.16939890710382513
                 ],
-                "bp": 0.9903382397772544,
-                "sys_len": 206,
+                "bp": 0.9657735711441044,
+                "sys_len": 201,
                 "ref_len": 208,
-                "sacrebleu": 0.2976938560971466,
-                "score": 0.2976938560971466,
+                "sacrebleu": 0.3083902088083731,
+                "score": 0.3083902088083731,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1737766619242665,
-                "score_ci_high": 0.44451335565361355,
-                "sacrebleu_ci_low": 0.1737766619242665,
-                "sacrebleu_ci_high": 0.44451335565361355
+                "score_ci_low": 0.19420808815274457,
+                "score_ci_high": 0.5045525648576851,
+                "sacrebleu_ci_low": 0.19420808815274457,
+                "sacrebleu_ci_high": 0.5045525648576851
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    168,
-                    133,
-                    105,
-                    84
+                    172,
+                    138,
+                    110,
+                    89
                 ],
                 "totals": [
-                    210,
-                    204,
-                    198,
-                    192
+                    211,
+                    205,
+                    199,
+                    193
                 ],
                 "precisions": [
-                    0.8,
-                    0.6519607843137255,
-                    0.5303030303030303,
-                    0.4375
+                    0.8151658767772512,
+                    0.673170731707317,
+                    0.5527638190954773,
+                    0.461139896373057
                 ],
                 "bp": 1.0,
-                "sys_len": 210,
+                "sys_len": 211,
                 "ref_len": 208,
-                "sacrebleu": 0.5897981509424008,
-                "score": 0.5897981509424008,
+                "sacrebleu": 0.6115555063363534,
+                "score": 0.6115555063363534,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.46597275289680684,
-                "score_ci_high": 0.6510065766151202,
-                "sacrebleu_ci_low": 0.46597275289680684,
-                "sacrebleu_ci_high": 0.6510065766151202
+                "score_ci_low": 0.45325441924272025,
+                "score_ci_high": 0.6571232984712668,
+                "sacrebleu_ci_low": 0.45325441924272025,
+                "sacrebleu_ci_high": 0.6571232984712668
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    160,
-                    111,
-                    78,
-                    55
+                    152,
+                    103,
+                    72,
+                    51
                 ],
                 "totals": [
-                    228,
-                    222,
-                    216,
-                    210
+                    225,
+                    219,
+                    213,
+                    207
                 ],
                 "precisions": [
-                    0.7017543859649124,
-                    0.5,
-                    0.36111111111111116,
-                    0.2619047619047619
+                    0.6755555555555556,
+                    0.4703196347031963,
+                    0.3380281690140845,
+                    0.24637681159420288
                 ],
                 "bp": 1.0,
-                "sys_len": 228,
+                "sys_len": 225,
                 "ref_len": 208,
-                "sacrebleu": 0.4268102558559915,
-                "score": 0.4268102558559915,
+                "sacrebleu": 0.4033218270083536,
+                "score": 0.4033218270083536,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.33894724398193377,
-                "score_ci_high": 0.5442092547992242,
-                "sacrebleu_ci_low": 0.33894724398193377,
-                "sacrebleu_ci_high": 0.5442092547992242
+                "score_ci_low": 0.30195935503031646,
+                "score_ci_high": 0.5476360215647604,
+                "sacrebleu_ci_low": 0.30195935503031646,
+                "sacrebleu_ci_high": 0.5476360215647604
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
                     147,
-                    98,
-                    65,
-                    46
+                    95,
+                    60,
+                    43
                 ],
                 "totals": [
-                    213,
-                    207,
-                    201,
-                    195
+                    214,
+                    208,
+                    202,
+                    196
                 ],
                 "precisions": [
-                    0.6901408450704225,
-                    0.47342995169082125,
-                    0.3233830845771144,
-                    0.23589743589743592
+                    0.6869158878504673,
+                    0.4567307692307692,
+                    0.29702970297029707,
+                    0.2193877551020408
                 ],
                 "bp": 1.0,
-                "sys_len": 213,
+                "sys_len": 214,
                 "ref_len": 208,
-                "sacrebleu": 0.3973365305222621,
-                "score": 0.3973365305222621,
+                "sacrebleu": 0.3781325158505603,
+                "score": 0.3781325158505603,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.32188371117377373,
-                "score_ci_high": 0.45496400690805394,
-                "sacrebleu_ci_low": 0.32188371117377373,
-                "sacrebleu_ci_high": 0.45496400690805394
+                "score_ci_low": 0.2752989922105636,
+                "score_ci_high": 0.5385214852105319,
+                "sacrebleu_ci_low": 0.2752989922105636,
+                "sacrebleu_ci_high": 0.5385214852105319
             },
-            "score": 0.41609323273138143,
+            "score": 0.41710792069047686,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.55858826250611,
+        "score": 0.5895740337134209,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }

results/bluebench/{2025-07-02T18-12-30_evaluation_results.json → 2025-07-03T16-05-29_evaluation_results.json}

@@ -1,6 +1,6 @@
 {
     "environment_info": {
-        "timestamp_utc": "2025-07-02T22:12:26.883897Z",
+        "timestamp_utc": "2025-07-03T20:05:25.384483Z",
         "command_line_invocation": [
             "/dccstor/jbworks/miniforge3/envs/bb/bin/unitxt-evaluate",
             "--tasks",
@@ -42,7 +42,7 @@
             "cache_dir": null
         },
         "unitxt_version": "1.25.0",
-        "unitxt_commit_hash": "c8a5e77fd6ca62039b915dc10700323f50ccaacf",
+        "unitxt_commit_hash": "f087fa0d9c77a2dab916bae59414b093e5be4041",
         "python_version": "3.10.18",
         "system": "Linux",
         "system_version": "#1 SMP PREEMPT_DYNAMIC Fri Aug 9 14:06:03 EDT 2024",
@@ -292,66 +292,66 @@
         "chatbot_abilities": {
             "arena_hard_generation_english_gpt_4_0314_reference": {
                 "num_of_instances": 100,
-                "llama_3_70b_instruct_template_arena_hard": 0.9156626506024096,
-                "score": 0.9156626506024096,
+                "llama_3_70b_instruct_template_arena_hard": 0.861271676300578,
+                "score": 0.861271676300578,
                 "score_name": "llama_3_70b_instruct_template_arena_hard"
             },
-            "score": 0.9156626506024096,
+            "score": 0.861271676300578,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "entity_extraction": {
             "universal_ner_en_ewt": {
                 "num_of_instances": 100,
-                "f1_Person": 0.1764705882352941,
-                "f1_Organization": 1.0,
-                "f1_Location": 0.06060606060606061,
-                "f1_macro": 0.41235888294711825,
-                "recall_macro": 0.057367149758454104,
-                "precision_macro": 0.12794612794612795,
-                "in_classes_support": 0.35443037974683544,
-                "f1_micro": 0.05194805194805195,
-                "recall_micro": 0.05333333333333334,
-                "precision_micro": 0.05063291139240506,
-                "score": 0.05194805194805195,
+                "f1_Person": 0.6956521739130435,
+                "f1_Organization": 0.6250000000000001,
+                "f1_Location": 0.744186046511628,
+                "f1_macro": 0.6882794068082237,
+                "recall_macro": 0.6922015182884748,
+                "precision_macro": 0.6977709975421647,
+                "in_classes_support": 0.8210526315789474,
+                "f1_micro": 0.611764705882353,
+                "recall_micro": 0.6933333333333334,
+                "precision_micro": 0.5473684210526316,
+                "score": 0.611764705882353,
                 "score_name": "f1_micro",
-                "score_ci_low": 0.012803676797310442,
-                "score_ci_high": 0.0936247158405427,
-                "f1_micro_ci_low": 0.012803676797310442,
-                "f1_micro_ci_high": 0.0936247158405427
+                "score_ci_low": 0.5125719093345596,
+                "score_ci_high": 0.6797127331701401,
+                "f1_micro_ci_low": 0.5125719093345596,
+                "f1_micro_ci_high": 0.6797127331701401
             },
-            "score": 0.05194805194805195,
+            "score": 0.611764705882353,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "knowledge": {
             "mmlu_pro_biology": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
-                "accuracy_ci_high": 1.0,
+                "accuracy": 0.5714285714285714,
+                "accuracy_ci_low": 0.14285714285714285,
+                "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
-                "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score": 0.5714285714285714,
+                "score_ci_high": 0.8571428571428571,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_business": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
             },
             "mmlu_pro_chemistry": {
-                "accuracy": 0.14285714285714285,
+                "accuracy": 0.0,
                 "accuracy_ci_low": 0.0,
-                "accuracy_ci_high": 0.5714285714285714,
+                "accuracy_ci_high": 0.0,
                 "score_name": "accuracy",
-                "score": 0.14285714285714285,
-                "score_ci_high": 0.5714285714285714,
+                "score": 0.0,
+                "score_ci_high": 0.0,
                 "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
@@ -386,11 +386,11 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_health": {
-                "accuracy": 0.5714285714285714,
+                "accuracy": 0.42857142857142855,
                 "accuracy_ci_low": 0.14285714285714285,
                 "accuracy_ci_high": 0.8571428571428571,
                 "score_name": "accuracy",
-                "score": 0.5714285714285714,
+                "score": 0.42857142857142855,
                 "score_ci_high": 0.8571428571428571,
                 "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 7
@@ -416,13 +416,13 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_math": {
-                "accuracy": 0.8571428571428571,
-                "accuracy_ci_low": 0.42857142857142855,
+                "accuracy": 1.0,
+                "accuracy_ci_low": 1.0,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.8571428571428571,
+                "score": 1.0,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.42857142857142855,
+                "score_ci_low": 1.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_other": {
@@ -436,277 +436,277 @@
                 "num_of_instances": 7
             },
             "mmlu_pro_philosophy": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.31927964061584246,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.31927964061584246,
                 "num_of_instances": 7
             },
             "mmlu_pro_physics": {
-                "accuracy": 0.42857142857142855,
-                "accuracy_ci_low": 0.14285714285714285,
-                "accuracy_ci_high": 0.8571428571428571,
+                "accuracy": 0.14285714285714285,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.5714285714285714,
                 "score_name": "accuracy",
-                "score": 0.42857142857142855,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.14285714285714285,
+                "score": 0.14285714285714285,
+                "score_ci_high": 0.5714285714285714,
+                "score_ci_low": 0.0,
                 "num_of_instances": 7
             },
             "mmlu_pro_psychology": {
-                "accuracy": 0.7142857142857143,
-                "accuracy_ci_low": 0.2857142857142857,
+                "accuracy": 0.8571428571428571,
+                "accuracy_ci_low": 0.42857142857142855,
                 "accuracy_ci_high": 1.0,
                 "score_name": "accuracy",
-                "score": 0.7142857142857143,
+                "score": 0.8571428571428571,
                 "score_ci_high": 1.0,
-                "score_ci_low": 0.2857142857142857,
+                "score_ci_low": 0.42857142857142855,
                 "num_of_instances": 7
             },
-            "score": 0.5816326530612245,
+            "score": 0.5510204081632653,
             "score_name": "subsets_mean",
             "num_of_instances": 98
         },
         "legal": {
             "legalbench_abercrombie": {
-                "f1_macro": 0.28095238095238095,
-                "f1_suggestive": 0.3333333333333333,
+                "f1_macro": 0.3327272727272727,
+                "f1_suggestive": 0.36363636363636365,
                 "f1_generic": 0.0,
-                "f1_fanciful": 0.0,
-                "f1_descriptive": 0.5714285714285714,
+                "f1_fanciful": 0.4,
+                "f1_descriptive": 0.4,
                 "f1_arbitrary": 0.5,
-                "f1_macro_ci_low": 0.13333333333333333,
-                "f1_macro_ci_high": 0.5745612934916534,
+                "f1_macro_ci_low": 0.15714285714285714,
+                "f1_macro_ci_high": 0.632306540058964,
                 "score_name": "f1_micro",
-                "score": 0.3333333333333333,
-                "score_ci_high": 0.5951377864663647,
-                "score_ci_low": 0.125,
+                "score": 0.35714285714285715,
+                "score_ci_high": 0.6204499589094157,
+                "score_ci_low": 0.14285714285714285,
                 "num_of_instances": 20,
                 "accuracy": 0.25,
                 "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.3333333333333333,
-                "f1_micro_ci_low": 0.125,
-                "f1_micro_ci_high": 0.5951377864663647
+                "accuracy_ci_high": 0.45,
+                "f1_micro": 0.35714285714285715,
+                "f1_micro_ci_low": 0.14285714285714285,
+                "f1_micro_ci_high": 0.6204499589094157
             },
             "legalbench_corporate_lobbying": {
-                "f1_macro": 0.2631578947368421,
-                "f1_no": 0.5263157894736842,
+                "f1_macro": 0.2,
+                "f1_no": 0.4,
                 "f1_yes": 0.0,
-                "f1_macro_ci_low": 0.10466984036978784,
-                "f1_macro_ci_high": 0.391304347826087,
+                "f1_macro_ci_low": 0.0625,
+                "f1_macro_ci_high": 0.3401444558302973,
                 "score_name": "f1_micro",
-                "score": 0.4,
-                "score_ci_high": 0.6666666666666666,
-                "score_ci_low": 0.18181818181818182,
+                "score": 0.3076923076923077,
+                "score_ci_high": 0.5517241379310345,
+                "score_ci_low": 0.08695652173913043,
                 "num_of_instances": 20,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.4,
-                "f1_micro_ci_low": 0.18181818181818182,
-                "f1_micro_ci_high": 0.6666666666666666
+                "accuracy": 0.2,
+                "accuracy_ci_low": 0.05,
+                "accuracy_ci_high": 0.4,
+                "f1_micro": 0.3076923076923077,
+                "f1_micro_ci_low": 0.08695652173913043,
+                "f1_micro_ci_high": 0.5517241379310345
             },
             "legalbench_function_of_decision_section": {
-                "f1_macro": 0.21428571428571427,
+                "f1_macro": 0.047619047619047616,
                 "f1_conclusion": 0.0,
                 "f1_decree": 0.0,
                 "f1_issue": 0.3333333333333333,
-                "f1_analysis": 0.6666666666666666,
+                "f1_analysis": 0.0,
                 "f1_facts": 0.0,
-                "f1_procedural history": 0.5,
+                "f1_procedural history": 0.0,
                 "f1_rule": 0.0,
-                "f1_macro_ci_low": 0.06666666666666667,
-                "f1_macro_ci_high": 0.4444444444444444,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.14285714285714285,
                 "score_name": "f1_micro",
-                "score": 0.2222222222222222,
-                "score_ci_high": 0.4666666666666667,
+                "score": 0.08695652173913043,
+                "score_ci_high": 0.3333333333333333,
                 "score_ci_low": 0.0,
                 "num_of_instances": 20,
-                "accuracy": 0.15,
-                "accuracy_ci_low": 0.05,
-                "accuracy_ci_high": 0.35,
-                "f1_micro": 0.2222222222222222,
+                "accuracy": 0.05,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.25,
+                "f1_micro": 0.08695652173913043,
                 "f1_micro_ci_low": 0.0,
-                "f1_micro_ci_high": 0.4666666666666667
+                "f1_micro_ci_high": 0.3333333333333333
             },
             "legalbench_international_citizenship_questions": {
-                "f1_macro": 0.39285714285714285,
-                "f1_yes": 0.5,
-                "f1_no": 0.2857142857142857,
-                "f1_macro_ci_low": 0.17424242424242425,
-                "f1_macro_ci_high": 0.6514486967849482,
+                "f1_macro": 0.15384615384615385,
+                "f1_yes": 0.3076923076923077,
+                "f1_no": 0.0,
+                "f1_macro_ci_low": 0.0,
+                "f1_macro_ci_high": 0.36363533071680015,
                 "score_name": "f1_micro",
-                "score": 0.38461538461538464,
-                "score_ci_high": 0.625,
-                "score_ci_low": 0.16666666666666666,
+                "score": 0.16,
+                "score_ci_high": 0.4444444444444444,
+                "score_ci_low": 0.0,
                 "num_of_instances": 20,
-                "accuracy": 0.25,
-                "accuracy_ci_low": 0.1,
-                "accuracy_ci_high": 0.5,
-                "f1_micro": 0.38461538461538464,
-                "f1_micro_ci_low": 0.16666666666666666,
-                "f1_micro_ci_high": 0.625
+                "accuracy": 0.1,
+                "accuracy_ci_low": 0.0,
+                "accuracy_ci_high": 0.3,
+                "f1_micro": 0.16,
+                "f1_micro_ci_low": 0.0,
+                "f1_micro_ci_high": 0.4444444444444444
             },
             "legalbench_proa": {
-                "f1_macro": 0.728744939271255,
-                "f1_yes": 0.6153846153846154,
-                "f1_no": 0.8421052631578947,
-                "f1_macro_ci_low": 0.4957866010638887,
-                "f1_macro_ci_high": 0.875,
+                "f1_macro": 0.8071428571428572,
+                "f1_yes": 0.7142857142857143,
+                "f1_no": 0.9,
+                "f1_macro_ci_low": 0.6074318256157948,
+                "f1_macro_ci_high": 0.9285714285714286,
                 "score_name": "f1_micro",
-                "score": 0.75,
-                "score_ci_high": 0.8571428571428571,
-                "score_ci_low": 0.5185185185185185,
+                "score": 0.8235294117647058,
+                "score_ci_high": 0.918918918918919,
+                "score_ci_low": 0.6206896551724138,
                 "num_of_instances": 20,
-                "accuracy": 0.6,
-                "accuracy_ci_low": 0.35,
-                "accuracy_ci_high": 0.75,
-                "f1_micro": 0.75,
-                "f1_micro_ci_low": 0.5185185185185185,
-                "f1_micro_ci_high": 0.8571428571428571
+                "accuracy": 0.7,
+                "accuracy_ci_low": 0.45,
+                "accuracy_ci_high": 0.85,
+                "f1_micro": 0.8235294117647058,
+                "f1_micro_ci_low": 0.6206896551724138,
+                "f1_micro_ci_high": 0.918918918918919
             },
-            "score": 0.418034188034188,
+            "score": 0.3470642196678002,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "news_classification": {
             "20_newsgroups_short": {
-                "f1_macro": 0.2970707070707071,
+                "f1_macro": 0.2502886002886003,
                 "f1_cars": 0.3333333333333333,
-                "f1_motorcycles": 0.4,
+                "f1_motorcycles": 0.2222222222222222,
                 "f1_windows x": 0.0,
                 "f1_atheism": 0.0,
-                "f1_religion": 0.2857142857142857,
+                "f1_religion": 0.0,
                 "f1_medicine": 0.8571428571428571,
                 "f1_christianity": 0.4,
-                "f1_computer graphics": 0.6666666666666666,
-                "f1_microsoft windows": 0.0,
+                "f1_computer graphics": 0.5454545454545454,
+                "f1_microsoft windows": 0.2857142857142857,
                 "f1_middle east": 0.2857142857142857,
-                "f1_pc hardware": 0.36363636363636365,
-                "f1_mac hardware": 0.2857142857142857,
+                "f1_pc hardware": 0.0,
+                "f1_mac hardware": 0.6,
                 "f1_for sale": 0.0,
                 "f1_guns": 0.0,
-                "f1_space": 0.3333333333333333,
+                "f1_space": 0.5714285714285714,
                 "f1_cryptography": 0.0,
-                "f1_electronics": 0.6666666666666666,
                 "f1_baseball": 0.2857142857142857,
                 "f1_hockey": 0.3333333333333333,
-                "f1_politics": 0.4444444444444444,
-                "f1_macro_ci_low": 0.23770175460525397,
-                "f1_macro_ci_high": 0.42938825894209254,
+                "f1_politics": 0.2857142857142857,
+                "f1_electronics": 0.0,
+                "f1_macro_ci_low": 0.18246110159300216,
+                "f1_macro_ci_high": 0.3487828715098917,
                 "score_name": "f1_micro",
-                "score": 0.3308270676691729,
-                "score_ci_high": 0.4412922379526243,
-                "score_ci_low": 0.23505462006400799,
+                "score": 0.2900763358778626,
+                "score_ci_high": 0.40298187323143425,
+                "score_ci_low": 0.1935483870967742,
                 "num_of_instances": 100,
-                "accuracy": 0.22,
-                "accuracy_ci_low": 0.15,
-                "accuracy_ci_high": 0.31,
-                "f1_micro": 0.3308270676691729,
-                "f1_micro_ci_low": 0.23505462006400799,
-                "f1_micro_ci_high": 0.4412922379526243
+                "accuracy": 0.19,
+                "accuracy_ci_low": 0.12,
+                "accuracy_ci_high": 0.28,
+                "f1_micro": 0.2900763358778626,
+                "f1_micro_ci_low": 0.1935483870967742,
+                "f1_micro_ci_high": 0.40298187323143425
             },
-            "score": 0.3308270676691729,
+            "score": 0.2900763358778626,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "product_help": {
             "cfpb_product_2023": {
-                "f1_macro": 0.5521822809958403,
-                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7627118644067796,
-                "f1_credit card or prepaid card": 0.5,
-                "f1_money transfer or virtual currency or money service": 0.5,
-                "f1_mortgage": 0.0,
-                "f1_debt collection": 0.6666666666666666,
-                "f1_checking or savings account": 0.7692307692307693,
-                "f1_payday loan or title loan or personal loan": 0.6666666666666666,
-                "f1_macro_ci_low": 0.3599760847493135,
-                "f1_macro_ci_high": 0.7238348271220593,
+                "f1_macro": 0.5571690214547357,
+                "f1_credit reporting or credit repair services or other personal consumer reports": 0.7428571428571429,
+                "f1_mortgage": 0.7692307692307693,
+                "f1_credit card or prepaid card": 0.4,
+                "f1_checking or savings account": 0.75,
+                "f1_debt collection": 0.5714285714285714,
+                "f1_student loan": 0.6666666666666666,
+                "f1_money transfer or virtual currency or money service": 0.0,
+                "f1_macro_ci_low": 0.4091725930188644,
+                "f1_macro_ci_high": 0.7840291796629795,
                 "score_name": "f1_micro",
-                "score": 0.7134502923976608,
-                "score_ci_high": 0.7943113493262172,
-                "score_ci_low": 0.622594308461465,
+                "score": 0.7125,
+                "score_ci_high": 0.7878787878787878,
+                "score_ci_low": 0.6122991744104558,
                 "num_of_instances": 100,
-                "accuracy": 0.61,
-                "accuracy_ci_low": 0.52,
-                "accuracy_ci_high": 0.71,
-                "f1_micro": 0.7134502923976608,
-                "f1_micro_ci_low": 0.622594308461465,
-                "f1_micro_ci_high": 0.7943113493262172
+                "accuracy": 0.57,
+                "accuracy_ci_low": 0.46,
+                "accuracy_ci_high": 0.67,
+                "f1_micro": 0.7125,
+                "f1_micro_ci_low": 0.6122991744104558,
+                "f1_micro_ci_high": 0.7878787878787878
             },
             "cfpb_product_watsonx": {
-                "f1_macro": 0.7025040319157967,
-                "f1_mortgages and loans": 0.5555555555555556,
-                "f1_credit card": 0.7619047619047619,
-                "f1_debt collection": 0.7058823529411765,
-                "f1_credit reporting": 0.7619047619047619,
-                "f1_retail banking": 0.7272727272727273,
-                "f1_macro_ci_low": 0.5612643049143822,
-                "f1_macro_ci_high": 0.8387759021003454,
+                "f1_macro": 0.6242213153203864,
+                "f1_mortgages and loans": 0.631578947368421,
+                "f1_credit card": 0.47058823529411764,
+                "f1_debt collection": 0.625,
+                "f1_credit reporting": 0.7272727272727273,
+                "f1_retail banking": 0.6666666666666666,
+                "f1_macro_ci_low": 0.4795206428546832,
+                "f1_macro_ci_high": 0.7772599736048471,
                 "score_name": "f1_micro",
-                "score": 0.7045454545454546,
-                "score_ci_high": 0.8222222222222222,
-                "score_ci_low": 0.5581395348837209,
+                "score": 0.627906976744186,
+                "score_ci_high": 0.7548100433549912,
+                "score_ci_low": 0.47619047619047616,
                 "num_of_instances": 50,
-                "accuracy": 0.62,
-                "accuracy_ci_low": 0.48,
-                "accuracy_ci_high": 0.76,
-                "f1_micro": 0.7045454545454546,
-                "f1_micro_ci_low": 0.5581395348837209,
-                "f1_micro_ci_high": 0.8222222222222222
+                "accuracy": 0.54,
+                "accuracy_ci_low": 0.4,
+                "accuracy_ci_high": 0.68,
+                "f1_micro": 0.627906976744186,
+                "f1_micro_ci_low": 0.47619047619047616,
+                "f1_micro_ci_high": 0.7548100433549912
             },
-            "score": 0.7089978734715576,
+            "score": 0.6702034883720931,
             "score_name": "subsets_mean",
             "num_of_instances": 150
         },
         "qa_finance": {
             "fin_qa": {
                 "num_of_instances": 100,
-                "program_accuracy": 0.22,
-                "score": 0.22,
+                "execution_accuracy": 0.19,
+                "program_accuracy": 0.19,
+                "score": 0.19,
                 "score_name": "program_accuracy",
-                "execution_accuracy": 0.22,
-                "program_accuracy_ci_low": 0.14,
-                "program_accuracy_ci_high": 0.31,
-                "score_ci_low": 0.14,
-                "score_ci_high": 0.31,
-                "execution_accuracy_ci_low": 0.15,
-                "execution_accuracy_ci_high": 0.32
+                "execution_accuracy_ci_low": 0.12,
+                "execution_accuracy_ci_high": 0.27,
+                "program_accuracy_ci_low": 0.12,
+                "program_accuracy_ci_high": 0.28,
+                "score_ci_low": 0.12,
+                "score_ci_high": 0.28
             },
-            "score": 0.22,
+            "score": 0.19,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "rag_general": {
             "rag_response_generation_clapnq": {
-                "precision": 0.5085272156185048,
-                "recall": 0.66080916917758,
-                "f1": 0.5287389555044663,
-                "precision_ci_low": 0.47382989299216594,
-                "precision_ci_high": 0.5466238200685761,
-                "recall_ci_low": 0.6232438959613603,
-                "recall_ci_high": 0.7017592032617559,
-                "f1_ci_low": 0.5012059859293033,
-                "f1_ci_high": 0.5631021478879368,
+                "precision": 0.5002718214509941,
+                "recall": 0.6396484528188586,
+                "f1": 0.5107793159153189,
+                "precision_ci_low": 0.46201748613798926,
+                "precision_ci_high": 0.5416393053134957,
+                "recall_ci_low": 0.5954333759828621,
+                "recall_ci_high": 0.6836164546156923,
+                "f1_ci_low": 0.47907009191115363,
+                "f1_ci_high": 0.5455987522626499,
                 "score_name": "f1",
-                "score": 0.5287389555044663,
-                "score_ci_high": 0.5631021478879368,
-                "score_ci_low": 0.5012059859293033,
+                "score": 0.5107793159153189,
+                "score_ci_high": 0.5455987522626499,
+                "score_ci_low": 0.47907009191115363,
                 "num_of_instances": 100,
-                "correctness_f1_bert_score.deberta_large_mnli": 0.7071846815943718,
-                "correctness_recall_bert_score.deberta_large_mnli": 0.7436953192949295,
-                "correctness_precision_bert_score.deberta_large_mnli": 0.6864294826984405,
-                "faithfullness_f1_token_overlap": 0.43295540530097154,
-                "faithfullness_recall_token_overlap": 0.3454106244145275,
-                "faithfullness_precision_token_overlap": 0.7364704084394016,
-                "correctness_f1_token_overlap": 0.5287389555044663,
-                "correctness_recall_token_overlap": 0.66080916917758,
-                "correctness_precision_token_overlap": 0.5085272156185048
+                "correctness_f1_bert_score.deberta_large_mnli": 0.6980990976095199,
+                "correctness_recall_bert_score.deberta_large_mnli": 0.7327082559466362,
+                "correctness_precision_bert_score.deberta_large_mnli": 0.6813026934862136,
+                "faithfullness_f1_token_overlap": 0.4166097840148595,
+                "faithfullness_recall_token_overlap": 0.33714582498686896,
+                "faithfullness_precision_token_overlap": 0.7097636487233601,
+                "correctness_f1_token_overlap": 0.5107793159153189,
+                "correctness_recall_token_overlap": 0.6396484528188586,
+                "correctness_precision_token_overlap": 0.5002718214509941
             },
-            "score": 0.5287389555044663,
+            "score": 0.5107793159153189,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
@@ -714,93 +714,93 @@
             "hellaswag": {
                 "accuracy": 0.57,
                 "accuracy_ci_low": 0.47,
-                "accuracy_ci_high": 0.66,
+                "accuracy_ci_high": 0.67,
                 "score_name": "accuracy",
                 "score": 0.57,
-                "score_ci_high": 0.66,
+                "score_ci_high": 0.67,
                 "score_ci_low": 0.47,
                 "num_of_instances": 100
             },
             "openbook_qa": {
-                "accuracy": 0.94,
-                "accuracy_ci_low": 0.88,
+                "accuracy": 0.93,
+                "accuracy_ci_low": 0.86,
                 "accuracy_ci_high": 0.97,
                 "score_name": "accuracy",
-                "score": 0.94,
+                "score": 0.93,
                 "score_ci_high": 0.97,
-                "score_ci_low": 0.88,
+                "score_ci_low": 0.86,
                 "num_of_instances": 100
             },
-            "score": 0.7549999999999999,
+            "score": 0.75,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
         "safety": {
             "attaq_500": {
-                "severity_critical": 0.0,
+                "severity_critical": 1.0,
                 "severity_high": 1.0,
-                "severity_medium": 1.0,
+                "severity_medium": 0.0,
                 "severity_low": 98.0,
-                "category_harmful_info": 0.84784276975644,
-                "category_deception": 0.9021891548176005,
-                "category_explicit_content": 0.8979561521549418,
-                "category_violence": 0.8355905269228276,
-                "category_discrimination": 0.9000429752327148,
-                "category_substance_abuse": 0.9063073645035425,
-                "category_pii": 0.8448789252175225,
-                "safety": 0.8615333879697654,
-                "safety_ci_low": 0.83436617059804,
-                "safety_ci_high": 0.8854245019985251,
+                "category_harmful_info": 0.8651335061702643,
+                "category_deception": 0.9000077115164863,
+                "category_explicit_content": 0.921519504502566,
+                "category_violence": 0.8719599247806602,
+                "category_discrimination": 0.890523836726234,
+                "category_substance_abuse": 0.8867026734683249,
+                "category_pii": 0.8656618578566445,
+                "safety": 0.8804330071429528,
+                "safety_ci_low": 0.854673207278556,
+                "safety_ci_high": 0.9011431885957104,
                 "score_name": "safety",
-                "score": 0.8615333879697654,
-                "score_ci_high": 0.8854245019985251,
-                "score_ci_low": 0.83436617059804,
+                "score": 0.8804330071429528,
+                "score_ci_high": 0.9011431885957104,
+                "score_ci_low": 0.854673207278556,
                 "num_of_instances": 100
             },
-            "score": 0.8615333879697654,
+            "score": 0.8804330071429528,
             "score_name": "subsets_mean",
             "num_of_instances": 100
         },
         "summarization": {
             "billsum_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.4040204017718249,
-                "rougeLsum": 0.35597114618020226,
-                "rougeL": 0.28083246354589236,
-                "score": 0.28083246354589236,
+                "rougeLsum": 0.3473556491516916,
+                "rouge1": 0.39926507000515976,
+                "rougeL": 0.2767560155158648,
+                "score": 0.2767560155158648,
                 "score_name": "rougeL",
-                "rouge2": 0.19424207857553685,
-                "rouge1_ci_low": 0.3804668414357713,
-                "rouge1_ci_high": 0.42440603302194135,
-                "rougeLsum_ci_low": 0.3330776446904628,
-                "rougeLsum_ci_high": 0.37423146520039924,
-                "rougeL_ci_low": 0.2631846690415355,
-                "rougeL_ci_high": 0.297661597023444,
-                "score_ci_low": 0.2631846690415355,
-                "score_ci_high": 0.297661597023444,
-                "rouge2_ci_low": 0.17875398333480141,
-                "rouge2_ci_high": 0.21045035388946662
+                "rouge2": 0.19274567237234305,
+                "rougeLsum_ci_low": 0.32485838908566234,
+                "rougeLsum_ci_high": 0.3693170716822161,
+                "rouge1_ci_low": 0.37601738221785536,
+                "rouge1_ci_high": 0.4231936969988669,
+                "rougeL_ci_low": 0.2579031827125714,
+                "rougeL_ci_high": 0.2956711521727504,
+                "score_ci_low": 0.2579031827125714,
+                "score_ci_high": 0.2956711521727504,
+                "rouge2_ci_low": 0.17696461232891966,
+                "rouge2_ci_high": 0.21036164102422508
             },
             "tldr_document_filtered_to_6000_chars": {
                 "num_of_instances": 100,
-                "rouge1": 0.11279720972972024,
-                "rougeLsum": 0.09286793656213359,
-                "rougeL": 0.08401612353653809,
-                "score": 0.08401612353653809,
+                "rougeLsum": 0.09554809497310719,
+                "rouge1": 0.11124842630083163,
+                "rougeL": 0.08810757028907336,
+                "score": 0.08810757028907336,
                 "score_name": "rougeL",
-                "rouge2": 0.015958528620851967,
-                "rouge1_ci_low": 0.09711088673606806,
-                "rouge1_ci_high": 0.129107538361123,
-                "rougeLsum_ci_low": 0.08064073960403449,
-                "rougeLsum_ci_high": 0.10547935967372425,
-                "rougeL_ci_low": 0.07274911157944615,
-                "rougeL_ci_high": 0.09426779388878173,
-                "score_ci_low": 0.07274911157944615,
-                "score_ci_high": 0.09426779388878173,
-                "rouge2_ci_low": 0.010760935565448007,
-                "rouge2_ci_high": 0.022872221463206353
+                "rouge2": 0.013953037020221,
+                "rougeLsum_ci_low": 0.08369233683015881,
+                "rougeLsum_ci_high": 0.10862661126964819,
+                "rouge1_ci_low": 0.09681920660757795,
+                "rouge1_ci_high": 0.12694608788162032,
+                "rougeL_ci_low": 0.07725516437043164,
+                "rougeL_ci_high": 0.09979175527135194,
+                "score_ci_low": 0.07725516437043164,
+                "score_ci_high": 0.09979175527135194,
+                "rouge2_ci_low": 0.009529333584287262,
+                "rouge2_ci_high": 0.01956333225374928
             },
-            "score": 0.18242429354121523,
+            "score": 0.18243179290246908,
             "score_name": "subsets_mean",
             "num_of_instances": 200
         },
@@ -808,196 +808,196 @@
             "mt_flores_101_ara_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    155,
-                    113,
-                    83,
-                    64
+                    154,
+                    112,
+                    82,
+                    62
                 ],
                 "totals": [
-                    210,
-                    204,
-                    198,
-                    192
+                    211,
+                    205,
+                    199,
+                    193
                 ],
                 "precisions": [
-                    0.7380952380952381,
-                    0.553921568627451,
-                    0.41919191919191917,
-                    0.33333333333333337
+                    0.7298578199052134,
+                    0.5463414634146342,
+                    0.4120603015075377,
+                    0.3212435233160622
                 ],
                 "bp": 1.0,
-                "sys_len": 210,
+                "sys_len": 211,
                 "ref_len": 208,
-                "sacrebleu": 0.4888921699740167,
-                "score": 0.4888921699740167,
+                "sacrebleu": 0.47931872554319865,
+                "score": 0.47931872554319865,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3798109415174125,
-                "score_ci_high": 0.6407268377282389,
-                "sacrebleu_ci_low": 0.3798109415174125,
-                "sacrebleu_ci_high": 0.6407268377282389
+                "score_ci_low": 0.24428289285956215,
+                "score_ci_high": 0.6593405113773021,
+                "sacrebleu_ci_low": 0.24428289285956215,
+                "sacrebleu_ci_high": 0.6593405113773021
             },
             "mt_flores_101_deu_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    135,
-                    74,
-                    41,
-                    28
+                    129,
+                    75,
+                    44,
+                    31
                 ],
                 "totals": [
-                    214,
-                    208,
-                    202,
-                    196
+                    209,
+                    203,
+                    197,
+                    191
                 ],
                 "precisions": [
-                    0.6308411214953271,
-                    0.3557692307692308,
-                    0.20297029702970296,
-                    0.14285714285714288
+                    0.6172248803827751,
+                    0.3694581280788177,
+                    0.2233502538071066,
+                    0.16230366492146597
                 ],
                 "bp": 1.0,
-                "sys_len": 214,
+                "sys_len": 209,
                 "ref_len": 208,
-                "sacrebleu": 0.2840244364432591,
-                "score": 0.2840244364432591,
+                "sacrebleu": 0.3015302283803927,
+                "score": 0.3015302283803927,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1611750157450563,
-                "score_ci_high": 0.38205893926378137,
-                "sacrebleu_ci_low": 0.1611750157450563,
-                "sacrebleu_ci_high": 0.38205893926378137
+                "score_ci_low": 0.21453472951260177,
+                "score_ci_high": 0.41180045577045343,
+                "sacrebleu_ci_low": 0.21453472951260177,
+                "sacrebleu_ci_high": 0.41180045577045343
             },
             "mt_flores_101_eng_ara": {
                 "num_of_instances": 6,
                 "counts": [
-                    126,
-                    75,
-                    48,
-                    28
+                    121,
+                    65,
+                    39,
+                    23
                 ],
                 "totals": [
-                    283,
-                    277,
-                    271,
-                    265
+                    202,
+                    196,
+                    190,
+                    184
                 ],
                 "precisions": [
-                    0.44522968197879853,
-                    0.27075812274368233,
-                    0.1771217712177122,
-                    0.10566037735849056
+                    0.599009900990099,
+                    0.33163265306122447,
+                    0.20526315789473684,
+                    0.125
                 ],
-                "bp": 1.0,
-                "sys_len": 283,
+                "bp": 0.9659400899805457,
+                "sys_len": 202,
                 "ref_len": 209,
-                "sacrebleu": 0.21794024107489876,
-                "score": 0.21794024107489876,
+                "sacrebleu": 0.2580942133850595,
+                "score": 0.2580942133850595,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.09217992990730309,
-                "score_ci_high": 0.3421677301380105,
-                "sacrebleu_ci_low": 0.09217992990730309,
-                "sacrebleu_ci_high": 0.3421677301380105
+                "score_ci_low": 0.16272022297334088,
+                "score_ci_high": 0.3899948456223761,
+                "sacrebleu_ci_low": 0.16272022297334088,
+                "sacrebleu_ci_high": 0.3899948456223761
             },
             "mt_flores_101_eng_deu": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    96,
-                    63,
-                    44
+                    148,
+                    94,
+                    62,
+                    43
                 ],
                 "totals": [
-                    225,
-                    219,
-                    213,
-                    207
+                    217,
+                    211,
+                    205,
+                    199
                 ],
                 "precisions": [
-                    0.6711111111111111,
-                    0.4383561643835616,
-                    0.29577464788732394,
-                    0.21256038647342992
+                    0.6820276497695852,
+                    0.44549763033175355,
+                    0.3024390243902439,
+                    0.21608040201005024
                 ],
                 "bp": 1.0,
-                "sys_len": 225,
+                "sys_len": 217,
                 "ref_len": 216,
-                "sacrebleu": 0.3687790201913159,
-                "score": 0.3687790201913159,
+                "sacrebleu": 0.3753834719910266,
+                "score": 0.3753834719910266,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2414136580858641,
-                "score_ci_high": 0.4964998309794481,
-                "sacrebleu_ci_low": 0.2414136580858641,
-                "sacrebleu_ci_high": 0.4964998309794481
+                "score_ci_low": 0.28218613578483775,
+                "score_ci_high": 0.4868021159471225,
+                "sacrebleu_ci_low": 0.28218613578483775,
+                "sacrebleu_ci_high": 0.4868021159471225
             },
             "mt_flores_101_eng_fra": {
                 "num_of_instances": 6,
                 "counts": [
-                    182,
+                    185,
                     140,
-                    108,
-                    86
+                    106,
+                    84
                 ],
                 "totals": [
-                    238,
-                    232,
-                    226,
-                    220
+                    237,
+                    231,
+                    225,
+                    219
                 ],
                 "precisions": [
-                    0.7647058823529411,
-                    0.603448275862069,
-                    0.4778761061946903,
-                    0.39090909090909093
+                    0.7805907172995781,
+                    0.6060606060606061,
+                    0.47111111111111115,
+                    0.3835616438356164
                 ],
                 "bp": 1.0,
-                "sys_len": 238,
+                "sys_len": 237,
                 "ref_len": 235,
-                "sacrebleu": 0.5418528876013806,
-                "score": 0.5418528876013806,
+                "sacrebleu": 0.5407225594670461,
+                "score": 0.5407225594670461,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.39701762116131134,
-                "score_ci_high": 0.6650077786342699,
-                "sacrebleu_ci_low": 0.39701762116131134,
-                "sacrebleu_ci_high": 0.6650077786342699
+                "score_ci_low": 0.41563184037164763,
+                "score_ci_high": 0.670285988225504,
+                "sacrebleu_ci_low": 0.41563184037164763,
+                "sacrebleu_ci_high": 0.670285988225504
             },
             "mt_flores_101_eng_kor": {
                 "num_of_instances": 6,
                 "counts": [
-                    168,
-                    100,
-                    67,
+                    161,
+                    94,
+                    64,
                     44
                 ],
                 "totals": [
-                    263,
-                    257,
-                    251,
-                    245
+                    282,
+                    276,
+                    270,
+                    264
                 ],
                 "precisions": [
-                    0.6387832699619772,
-                    0.3891050583657587,
-                    0.26693227091633465,
-                    0.17959183673469387
+                    0.5709219858156028,
+                    0.3405797101449275,
+                    0.23703703703703702,
+                    0.16666666666666669
                 ],
                 "bp": 1.0,
-                "sys_len": 263,
+                "sys_len": 282,
                 "ref_len": 249,
-                "sacrebleu": 0.33039009651517476,
-                "score": 0.33039009651517476,
+                "sacrebleu": 0.2960500403923138,
+                "score": 0.2960500403923138,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.27101151031230714,
-                "score_ci_high": 0.36483020674748473,
-                "sacrebleu_ci_low": 0.27101151031230714,
-                "sacrebleu_ci_high": 0.36483020674748473
+                "score_ci_low": 0.20900711841765263,
+                "score_ci_high": 0.3664266992038485,
+                "sacrebleu_ci_low": 0.20900711841765263,
+                "sacrebleu_ci_high": 0.3664266992038485
             },
             "mt_flores_101_eng_por": {
                 "num_of_instances": 6,
                 "counts": [
-                    176,
-                    130,
-                    104,
-                    85
+                    177,
+                    132,
+                    106,
+                    87
                 ],
                 "totals": [
                     224,
@@ -1006,275 +1006,275 @@
                     206
                 ],
                 "precisions": [
-                    0.7857142857142857,
-                    0.5963302752293578,
-                    0.49056603773584906,
-                    0.41262135922330095
+                    0.7901785714285714,
+                    0.6055045871559632,
+                    0.5,
+                    0.4223300970873787
                 ],
                 "bp": 1.0,
                 "sys_len": 224,
                 "ref_len": 222,
-                "sacrebleu": 0.5549452941211316,
-                "score": 0.5549452941211316,
+                "sacrebleu": 0.5637884578677731,
+                "score": 0.5637884578677731,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.5080385803050895,
-                "score_ci_high": 0.6056385535160684,
-                "sacrebleu_ci_low": 0.5080385803050895,
-                "sacrebleu_ci_high": 0.6056385535160684
+                "score_ci_low": 0.489928470426138,
+                "score_ci_high": 0.680433063758059,
+                "sacrebleu_ci_low": 0.489928470426138,
+                "sacrebleu_ci_high": 0.680433063758059
             },
             "mt_flores_101_eng_ron": {
                 "num_of_instances": 6,
                 "counts": [
-                    161,
-                    112,
-                    80,
-                    61
+                    164,
+                    117,
+                    88,
+                    68
                 ],
                 "totals": [
-                    226,
-                    220,
-                    214,
-                    208
+                    230,
+                    224,
+                    218,
+                    212
                 ],
                 "precisions": [
-                    0.7123893805309734,
-                    0.509090909090909,
-                    0.37383177570093457,
-                    0.2932692307692308
+                    0.7130434782608696,
+                    0.5223214285714285,
+                    0.4036697247706422,
+                    0.32075471698113206
                 ],
-                "bp": 0.9824565942999044,
-                "sys_len": 226,
+                "bp": 1.0,
+                "sys_len": 230,
                 "ref_len": 230,
-                "sacrebleu": 0.43870970830172057,
-                "score": 0.43870970830172057,
+                "sacrebleu": 0.4686118552227835,
+                "score": 0.4686118552227835,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.33194220485772075,
-                "score_ci_high": 0.58132339989516,
-                "sacrebleu_ci_low": 0.33194220485772075,
-                "sacrebleu_ci_high": 0.58132339989516
+                "score_ci_low": 0.37119566818911415,
+                "score_ci_high": 0.6009886986216086,
+                "sacrebleu_ci_low": 0.37119566818911415,
+                "sacrebleu_ci_high": 0.6009886986216086
             },
             "mt_flores_101_eng_spa": {
                 "num_of_instances": 6,
                 "counts": [
-                    162,
-                    96,
-                    59,
-                    38
+                    165,
+                    99,
+                    62,
+                    40
                 ],
                 "totals": [
-                    232,
-                    226,
-                    220,
-                    214
+                    240,
+                    234,
+                    228,
+                    222
                 ],
                 "precisions": [
-                    0.6982758620689655,
-                    0.4247787610619469,
-                    0.2681818181818182,
-                    0.17757009345794394
+                    0.6875,
+                    0.4230769230769231,
+                    0.27192982456140347,
+                    0.1801801801801802
                 ],
-                "bp": 0.9536926844755759,
-                "sys_len": 232,
+                "bp": 0.9875778004938814,
+                "sys_len": 240,
                 "ref_len": 243,
-                "sacrebleu": 0.3287802305742193,
-                "score": 0.3287802305742193,
+                "sacrebleu": 0.3412206404201496,
+                "score": 0.3412206404201496,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.26331151782962714,
-                "score_ci_high": 0.41514044654110027,
-                "sacrebleu_ci_low": 0.26331151782962714,
-                "sacrebleu_ci_high": 0.41514044654110027
+                "score_ci_low": 0.2635289110445265,
+                "score_ci_high": 0.4271398025964264,
+                "sacrebleu_ci_low": 0.2635289110445265,
+                "sacrebleu_ci_high": 0.4271398025964264
             },
             "mt_flores_101_fra_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    160,
-                    111,
-                    76,
-                    52
+                    166,
+                    124,
+                    95,
+                    74
                 ],
                 "totals": [
-                    235,
-                    229,
-                    223,
-                    217
+                    217,
+                    211,
+                    205,
+                    199
                 ],
                 "precisions": [
-                    0.6808510638297872,
-                    0.4847161572052402,
-                    0.3408071748878924,
-                    0.23963133640552997
+                    0.7649769585253456,
+                    0.5876777251184834,
+                    0.4634146341463415,
+                    0.37185929648241206
                 ],
                 "bp": 1.0,
-                "sys_len": 235,
+                "sys_len": 217,
                 "ref_len": 208,
-                "sacrebleu": 0.4051799896055395,
-                "score": 0.4051799896055395,
+                "sacrebleu": 0.5275747391751492,
+                "score": 0.5275747391751492,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.30963965039484914,
-                "score_ci_high": 0.48975983948164015,
-                "sacrebleu_ci_low": 0.30963965039484914,
-                "sacrebleu_ci_high": 0.48975983948164015
+                "score_ci_low": 0.42226915386816166,
+                "score_ci_high": 0.5943948476687988,
+                "sacrebleu_ci_low": 0.42226915386816166,
+                "sacrebleu_ci_high": 0.5943948476687988
             },
             "mt_flores_101_jpn_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    130,
-                    78,
-                    51,
-                    35
+                    124,
+                    70,
+                    43,
+                    29
                 ],
                 "totals": [
-                    201,
-                    195,
-                    189,
-                    183
+                    203,
+                    197,
+                    191,
+                    185
                 ],
                 "precisions": [
-                    0.6467661691542288,
-                    0.4,
-                    0.2698412698412698,
-                    0.1912568306010929
+                    0.6108374384236454,
+                    0.3553299492385787,
+                    0.22513089005235604,
+                    0.15675675675675677
                 ],
-                "bp": 0.9657735711441044,
-                "sys_len": 201,
+                "bp": 0.9756703147754899,
+                "sys_len": 203,
                 "ref_len": 208,
-                "sacrebleu": 0.32829061667550713,
-                "score": 0.32829061667550713,
+                "sacrebleu": 0.28864085101108844,
+                "score": 0.28864085101108844,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.2181869562419046,
-                "score_ci_high": 0.40880949111808457,
-                "sacrebleu_ci_low": 0.2181869562419046,
-                "sacrebleu_ci_high": 0.40880949111808457
+                "score_ci_low": 0.1597891185828086,
+                "score_ci_high": 0.41067608209503315,
+                "sacrebleu_ci_low": 0.1597891185828086,
+                "sacrebleu_ci_high": 0.41067608209503315
             },
             "mt_flores_101_kor_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    138,
-                    78,
-                    51,
+                    135,
+                    81,
+                    52,
                     36
                 ],
                 "totals": [
-                    235,
-                    229,
-                    223,
-                    217
+                    227,
+                    221,
+                    215,
+                    209
                 ],
                 "precisions": [
-                    0.5872340425531914,
-                    0.3406113537117904,
-                    0.22869955156950675,
-                    0.16589861751152074
+                    0.5947136563876652,
+                    0.3665158371040724,
+                    0.24186046511627907,
+                    0.17224880382775118
                 ],
                 "bp": 1.0,
-                "sys_len": 235,
+                "sys_len": 227,
                 "ref_len": 208,
-                "sacrebleu": 0.2951512359070574,
-                "score": 0.2951512359070574,
+                "sacrebleu": 0.3086955995864012,
+                "score": 0.3086955995864012,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.1912990542612514,
-                "score_ci_high": 0.4880380677303778,
-                "sacrebleu_ci_low": 0.1912990542612514,
-                "sacrebleu_ci_high": 0.4880380677303778
+                "score_ci_low": 0.20211606137918547,
+                "score_ci_high": 0.46728840335394556,
+                "sacrebleu_ci_low": 0.20211606137918547,
+                "sacrebleu_ci_high": 0.46728840335394556
             },
             "mt_flores_101_por_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    170,
-                    124,
-                    91,
-                    69
+                    171,
+                    132,
+                    106,
+                    87
                 ],
                 "totals": [
-                    220,
-                    214,
-                    208,
-                    202
+                    261,
+                    255,
+                    249,
+                    243
                 ],
                 "precisions": [
-                    0.7727272727272727,
-                    0.5794392523364487,
-                    0.4375,
-                    0.3415841584158416
+                    0.6551724137931035,
+                    0.5176470588235295,
+                    0.42570281124497994,
+                    0.35802469135802467
                 ],
                 "bp": 1.0,
-                "sys_len": 220,
+                "sys_len": 261,
                 "ref_len": 208,
-                "sacrebleu": 0.5086013197257839,
-                "score": 0.5086013197257839,
+                "sacrebleu": 0.4768175289794086,
+                "score": 0.4768175289794086,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.3866475374245657,
-                "score_ci_high": 0.627503873315733,
-                "sacrebleu_ci_low": 0.3866475374245657,
-                "sacrebleu_ci_high": 0.627503873315733
+                "score_ci_low": 0.3030992641082691,
+                "score_ci_high": 0.6309096812158492,
+                "sacrebleu_ci_low": 0.3030992641082691,
+                "sacrebleu_ci_high": 0.6309096812158492
             },
             "mt_flores_101_ron_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    151,
-                    102,
-                    69,
-                    50
+                    155,
+                    104,
+                    73,
+                    56
                 ],
                 "totals": [
-                    274,
-                    268,
-                    262,
-                    256
+                    244,
+                    238,
+                    232,
+                    226
                 ],
                 "precisions": [
-                    0.551094890510949,
-                    0.3805970149253731,
-                    0.2633587786259542,
-                    0.1953125
+                    0.6352459016393442,
+                    0.4369747899159664,
+                    0.3146551724137931,
+                    0.24778761061946902
                 ],
                 "bp": 1.0,
-                "sys_len": 274,
+                "sys_len": 244,
                 "ref_len": 208,
-                "sacrebleu": 0.32228671229839423,
-                "score": 0.32228671229839423,
+                "sacrebleu": 0.3835554687282761,
+                "score": 0.3835554687282761,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.23866420296537516,
-                "score_ci_high": 0.5275923196288512,
-                "sacrebleu_ci_low": 0.23866420296537516,
-                "sacrebleu_ci_high": 0.5275923196288512
+                "score_ci_low": 0.30587174504684356,
+                "score_ci_high": 0.5458926535949988,
+                "sacrebleu_ci_low": 0.30587174504684356,
+                "sacrebleu_ci_high": 0.5458926535949988
             },
             "mt_flores_101_spa_eng": {
                 "num_of_instances": 6,
                 "counts": [
-                    145,
-                    89,
-                    60,
+                    146,
+                    93,
+                    64,
                     42
                 ],
                 "totals": [
-                    219,
-                    213,
-                    207,
-                    201
+                    222,
+                    216,
+                    210,
+                    204
                 ],
                 "precisions": [
-                    0.6621004566210046,
-                    0.4178403755868545,
-                    0.2898550724637681,
-                    0.20895522388059704
+                    0.6576576576576576,
+                    0.4305555555555556,
+                    0.30476190476190473,
+                    0.2058823529411765
                 ],
                 "bp": 1.0,
-                "sys_len": 219,
+                "sys_len": 222,
                 "ref_len": 208,
-                "sacrebleu": 0.3597842164302771,
-                "score": 0.3597842164302771,
+                "sacrebleu": 0.3650919189357931,
+                "score": 0.3650919189357931,
                 "score_name": "sacrebleu",
-                "score_ci_low": 0.25650430080602765,
-                "score_ci_high": 0.4279461456140117,
-                "sacrebleu_ci_low": 0.25650430080602765,
-                "sacrebleu_ci_high": 0.4279461456140117
+                "score_ci_low": 0.2728459331802588,
+                "score_ci_high": 0.49282040986442494,
+                "sacrebleu_ci_low": 0.2728459331802588,
+                "sacrebleu_ci_high": 0.49282040986442494
             },
-            "score": 0.3849072116959784,
+            "score": 0.3983397532723907,
             "score_name": "subsets_mean",
             "num_of_instances": 90
         },
-        "score": 0.5314925617842308,
+        "score": 0.5548524363995425,
         "score_name": "subsets_mean",
         "num_of_instances": 1537
     }