fix(sdk): strip review assignment metadata from eval rows

py/src/braintrust/framework.py

@@ -72,6 +72,11 @@
 Output = TypeVar("Output")
 Expected = TypeVar("Expected")
 
+REVIEW_ASSIGNMENT_METADATA_KEYS = {
+    "~__bt_assignments",
+    "~__bt_review_lists",
+}
+
 
 # https://stackoverflow.com/questions/287871/how-do-i-print-colored-text-to-the-terminal
 class bcolors:
@@ -1254,6 +1259,12 @@ def parameters(self) -> ValidatedParameters | None:
         return self._parameters
 
 
+def _strip_review_assignment_metadata(metadata: Metadata | None) -> Metadata:
+    if not metadata:
+        return {}
+    return {k: v for k, v in metadata.items() if k not in REVIEW_ASSIGNMENT_METADATA_KEYS}
+
+
 def init_experiment(
     project_name: str | None = None, experiment_name: str | None = None, set_current: bool = False, **kwargs: Any
 ) -> Experiment:
@@ -1609,7 +1620,7 @@ async def run_evaluator_task(datum, trial_index=0):
         if isinstance(datum, dict):
             datum = EvalCase.from_dict(datum)
 
-        metadata = {**(datum.metadata or {})}
+        metadata = _strip_review_assignment_metadata(datum.metadata)
         output = None
         error = None
         exc_info = None

py/src/braintrust/test_framework.py

@@ -226,6 +226,42 @@ def exact_match(input_value, output, expected):
     assert result.summary.scores["exact_match"].score == 1.0
 
 
+@pytest.mark.asyncio
+async def test_run_evaluator_strips_review_assignment_metadata():
+    seen_metadata = None
+
+    def task(input_value, hooks):
+        nonlocal seen_metadata
+        seen_metadata = dict(hooks.metadata)
+        return input_value * 2
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-evaluator",
+        data=[
+            EvalCase(
+                input=1,
+                metadata={
+                    "keep": "yes",
+                    "~__bt_assignments": ["user-id"],
+                    "~__bt_review_lists": {
+                        "__bt_default_review_list": {"status": "PENDING"},
+                    },
+                },
+            )
+        ],
+        task=task,
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+    )
+
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    assert seen_metadata == {"keep": "yes"}
+    assert result.results[0].metadata == {"keep": "yes"}
+
+
 @pytest.mark.asyncio
 async def test_eval_case_id_and_tags_are_passed_to_scorers():
     scorer_args = None