Fix: don't add positional bias check if it is disabled

IBM · Dec 18, 2024 · 62f95e6 · 62f95e6
1 parent a6d668b
commit 62f95e6
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 11 deletions.
diff --git a/examples/evaluate_existing_dataset_by_llm_as_judge_eval_assist.py b/examples/evaluate_existing_dataset_by_llm_as_judge_eval_assist.py
@@ -23,8 +23,8 @@
 dataset = load_dataset(
     card="cards.squad",
     metrics=metrics,
-    loader_limit=100,
-    max_test_instances=100,
+    loader_limit=10,
+    max_test_instances=10,
     split="test",
 )
 

diff --git a/src/unitxt/eval_assist_llm_as_judge_direct.py b/src/unitxt/eval_assist_llm_as_judge_direct.py
@@ -135,21 +135,27 @@ def get_results(
                     "assessment": assessment_prompts[i],
                     "positional_bias_assessment": assessment_prompts[
                         evaluations_count + i
-                    ],
+                    ]
+                    if self.check_positional_bias
+                    else None,
                     "summarization": summarization_prompts[i]
                     if self.generate_summaries
                     else None,
                     "option_selection": option_selection_prompts[i],
                     "posional_bias_option_selection": option_selection_prompts[
                         i + evaluations_count
-                    ],
+                    ]
+                    if self.check_positional_bias
+                    else None,
                 }
                 if self.include_prompts_in_result
                 else None,
                 "option_selection_completion": option_selection_outputs[i],
                 "positional_bias_option_selection_completion": option_selection_outputs[
                     evaluations_count + i
-                ],
+                ]
+                if self.check_positional_bias
+                else None,
                 "criteria": criterias[i].to_json(),
             }
             for i in range(evaluations_count)

diff --git a/src/unitxt/eval_assist_llm_as_judge_pairwise.py b/src/unitxt/eval_assist_llm_as_judge_pairwise.py
@@ -161,12 +161,6 @@ def get_instance_results(
             per_response_results[response_name_2]["assessments"].append(
                 assessment_outputs[i]
             )
-            per_response_results[response_name_1]["positional_bias_assessments"].append(
-                assessment_outputs[positional_bias_i]
-            )
-            per_response_results[response_name_2]["positional_bias_assessments"].append(
-                assessment_outputs[positional_bias_i]
-            )
             per_response_results[response_name_1]["selections"].append(
                 selected_response_name
             )
@@ -214,6 +208,12 @@ def get_instance_results(
 
             ## add positional bias
             if self.check_positional_bias:
+                per_response_results[response_name_1]["positional_bias_assessments"].append(
+                    assessment_outputs[positional_bias_i]
+                )
+                per_response_results[response_name_2]["positional_bias_assessments"].append(
+                    assessment_outputs[positional_bias_i]
+                )
                 positional_bias = selections[i] != selections[positional_bias_i]
 
                 per_response_results[response_name_1]["positional_bias"].append(