fix(evaluation): drop redundant duplicate_at_runtime flag so v2 runs work without manual DB patching

backend/app/crud/evaluations/dataset.py

@@ -33,10 +33,6 @@
 DATASET_META_ORIGINAL_ITEMS = "original_items_count"
 DATASET_META_TOTAL_ITEMS = "total_items_count"
 DATASET_META_DUPLICATION_FACTOR = "duplication_factor"
-# v2 marker: the stored CSV holds only the original rows and duplication is applied
-# at run time. Absent/false means the S3 data is already physically duplicated (v1),
-# so the run reads it as-is and must not multiply again.
-DATASET_META_DUPLICATE_AT_RUNTIME = "duplicate_at_runtime"
 
 
 def create_evaluation_dataset(

backend/app/services/evaluations/dataset.py

@@ -12,7 +12,6 @@
     upload_dataset_to_langfuse,
 )
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     DATASET_META_ORIGINAL_ITEMS,
     DATASET_META_TOTAL_ITEMS,
@@ -162,9 +161,6 @@ def upload_dataset(
         DATASET_META_TOTAL_ITEMS: total_items_count,
         DATASET_META_DUPLICATION_FACTOR: duplication_factor,
     }
-    if not use_langfuse:
-        # The stored CSV holds only the original rows; the run expands them.
-        metadata[DATASET_META_DUPLICATE_AT_RUNTIME] = True
 
     dataset = create_evaluation_dataset(
         session=session,

backend/app/services/evaluations/fast.py

@@ -29,7 +29,6 @@
 from app.crud.evaluations.batch import fetch_dataset_items
 from app.crud.evaluations.core import update_evaluation_run
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     download_csv_from_object_store,
 )
@@ -72,14 +71,13 @@ def load_run_dataset_items(
 
     - Langfuse-backed dataset (v1): items are already physically duplicated in
       Langfuse; read as-is via `fetch_dataset_items`, never re-multiplied.
-    - S3-only dataset (v2, `langfuse_dataset_id` NULL): download the original-items
-      CSV and, when the dataset is marked for run-time duplication, expand each row
-      ×duplication_factor with a unique item id per copy.
+    - S3-only dataset (`langfuse_dataset_id` NULL): download the original-items CSV
+      and expand each original row ×duplication_factor with a unique item id per copy.
 
     Both the fan-out sizing and the per-chunk load call this, so they agree on the
     same expanded item set. `duplication_factor`, when set, overrides the dataset's
-    stored factor for runtime-duplicated datasets only (see
-    `_load_items_from_object_store`); it never applies to a Langfuse dataset.
+    stored factor (see `_load_items_from_object_store`); it never applies to a
+    Langfuse dataset.
     """
     if dataset.langfuse_dataset_id:
         if langfuse is None:
@@ -102,10 +100,10 @@ def _load_items_from_object_store(
 ) -> list[dict[str, Any]]:
     """Parse the dataset's original-items CSV from S3 into fast-pipeline items.
 
-    When the dataset is marked for run-time duplication (v2), each original row is
-    emitted `duplication_factor` times with a distinct item id (`item_{row}_{dup}`)
-    so per-row score keys stay unique. A v1 dataset's S3 CSV is already physically
-    duplicated, so it loads as-is (factor forced to 1)."""
+    This loader is only reached for S3-only datasets (`langfuse_dataset_id` NULL),
+    whose stored CSV always holds the original (un-duplicated) rows. Each original
+    row is emitted `duplication_factor` times with a distinct item id
+    (`item_{row}_{dup}`) so per-row score keys stay unique."""
     if not dataset.object_store_url:
         raise ValueError(f"Dataset {dataset.id} has no object-store CSV to load")
 
@@ -116,13 +114,12 @@ def _load_items_from_object_store(
     original_items = parse_csv_items(csv_content)
 
     metadata = dataset.dataset_metadata or {}
-    duplicate_at_runtime = bool(metadata.get(DATASET_META_DUPLICATE_AT_RUNTIME, False))
     effective_factor = (
         duplication_factor
         if duplication_factor is not None
         else int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1))
     )
-    duplication_factor = max(1, effective_factor) if duplicate_at_runtime else 1
+    duplication_factor = max(1, effective_factor)
 
     items: list[dict[str, Any]] = []
     for row_idx, item in enumerate(original_items):
@@ -169,8 +166,8 @@ def validate_fast_evaluation_inputs(
     3. Dataset's original_items_count <= EVAL_FAST_MAX_UNIQUE_ROWS.
 
     `duplication_factor`, when provided, overrides the dataset's stored factor for
-    this run only and is supported for runtime-duplicated (v2) datasets exclusively
-    (rejected with 422 otherwise).
+    this run only and is supported for S3-only datasets exclusively; it is rejected
+    with 422 for Langfuse-backed datasets (whose items come pre-duplicated).
     """
     # 1. Dataset must exist (Langfuse id required for v1 runs only; see below).
     dataset = get_dataset_by_id(
@@ -198,14 +195,12 @@ def validate_fast_evaluation_inputs(
             ),
         )
 
-    if duplication_factor is not None and not (dataset.dataset_metadata or {}).get(
-        DATASET_META_DUPLICATE_AT_RUNTIME
-    ):
+    if duplication_factor is not None and dataset.langfuse_dataset_id:
         raise HTTPException(
             status_code=422,
             detail=(
-                f"{ERR_DUPLICATION_FACTOR_NOT_SUPPORTED}: this dataset is not "
-                "runtime-duplicated; re-upload to change its factor"
+                f"{ERR_DUPLICATION_FACTOR_NOT_SUPPORTED}: this dataset is "
+                "Langfuse-backed and pre-duplicated; re-upload to change its factor"
             ),
         )
 
@@ -291,8 +286,8 @@ def validate_and_start_fast_evaluation(
     it stays NULL and no webhook fires.
 
     `duplication_factor`, when provided, overrides the dataset's stored factor for
-    this run only and is supported for runtime-duplicated (v2) datasets exclusively
-    (rejected with 422 otherwise).
+    this run only and is supported for S3-only datasets exclusively; it is rejected
+    with 422 for Langfuse-backed datasets (whose items come pre-duplicated).
     """
     logger.info(
         f"[validate_and_start_fast_evaluation] Starting fast eval | "

backend/app/tests/api/routes/test_evaluation_dataset_v2.py

@@ -5,8 +5,8 @@
 
 - FR-19: 200, row created with null langfuse id, CSV stored in S3, Langfuse never
   called.
-- FR-20: response + persisted metadata carry the run-time-duplication marker and
-  original/total item counts.
+- FR-20: response + persisted metadata carry the original/total item counts and
+  the stored duplication factor.
 
 Object storage and Langfuse are the external boundaries and are mocked; the
 dataset row lands in the real (transactional) DB.
@@ -20,7 +20,6 @@
 
 from app.core.config import settings
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     DATASET_META_ORIGINAL_ITEMS,
     DATASET_META_TOTAL_ITEMS,
@@ -45,7 +44,7 @@ def test_upload_creates_langfuse_free_dataset(
         user_api_key_header: dict[str, str],
         db: Session,
     ) -> None:
-        """FR-19/FR-20: 200, null langfuse id, S3 stored, run-time-dup metadata."""
+        """FR-19/FR-20: 200, null langfuse id, S3 stored, factor/count metadata."""
         name = f"v2-route-{random_lower_string()}"
         with (
             patch(f"{_DATASET}.get_cloud_storage", return_value=MagicMock()),
@@ -79,7 +78,6 @@ def test_upload_creates_langfuse_free_dataset(
         assert persisted is not None
         assert persisted.langfuse_dataset_id is None
         meta = persisted.dataset_metadata
-        assert meta[DATASET_META_DUPLICATE_AT_RUNTIME] is True
         assert meta[DATASET_META_DUPLICATION_FACTOR] == 5
         assert meta[DATASET_META_ORIGINAL_ITEMS] == 3
         assert meta[DATASET_META_TOTAL_ITEMS] == 15

backend/app/tests/api/routes/test_evaluation_v2.py

@@ -18,7 +18,6 @@
 
 from app.core.config import settings
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     DATASET_META_ORIGINAL_ITEMS,
     DATASET_META_TOTAL_ITEMS,
@@ -49,7 +48,7 @@ def _make_dataset(*, db: Session, user_api_key: TestAuthContext) -> EvaluationDa
 def _make_runtime_dup_dataset(
     *, db: Session, user_api_key: TestAuthContext, duplication_factor: int = 5
 ) -> EvaluationDataset:
-    """A v2 runtime-duplicated dataset: null Langfuse id, S3 url, runtime marker."""
+    """A v2 runtime-duplicated dataset: null Langfuse id, S3 url, stored factor."""
     original = 3
     return create_evaluation_dataset(
         session=db,
@@ -58,7 +57,6 @@ def _make_runtime_dup_dataset(
             DATASET_META_ORIGINAL_ITEMS: original,
             DATASET_META_TOTAL_ITEMS: original * duplication_factor,
             DATASET_META_DUPLICATION_FACTOR: duplication_factor,
-            DATASET_META_DUPLICATE_AT_RUNTIME: True,
         },
         object_store_url="s3://bucket/datasets/v2.csv",
         langfuse_dataset_id=None,

backend/app/tests/services/evaluations/test_dataset_v2.py

@@ -6,7 +6,7 @@
 - FR-19: creates the `evaluation_dataset` row with `langfuse_dataset_id` null and
   never touches the Langfuse client.
 - FR-20: stores only the original rows (no physical duplication) and records the
-  run-time-duplication metadata.
+  original/total counts and stored duplication factor.
 
 Object storage is the external boundary and is mocked; the dataset row lands in
 the real (transactional) DB.
@@ -19,7 +19,6 @@
 from sqlmodel import Session
 
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     DATASET_META_ORIGINAL_ITEMS,
     DATASET_META_TOTAL_ITEMS,
@@ -68,10 +67,10 @@ def test_creates_langfuse_free_row_without_calling_langfuse(
         assert persisted.langfuse_dataset_id is None
         assert persisted.object_store_url == "s3://bucket/datasets/v2.csv"
 
-    def test_stores_original_rows_and_runtime_dup_metadata(
+    def test_stores_original_rows_and_factor_metadata(
         self, db: Session, user_api_key: TestAuthContext
     ) -> None:
-        """FR-20: original CSV stored verbatim; metadata records run-time dup."""
+        """FR-20: original CSV stored verbatim; metadata records the stored factor."""
         name = f"v2-meta-{random_lower_string()}"
         with (
             patch(f"{_DATASET}.get_cloud_storage", return_value=MagicMock()),
@@ -97,7 +96,8 @@ def test_stores_original_rows_and_runtime_dup_metadata(
         assert mock_upload.call_args.kwargs["csv_content"] == _CSV
 
         meta = dataset.dataset_metadata
-        assert meta[DATASET_META_DUPLICATE_AT_RUNTIME] is True
+        # The removed run-time-duplication flag must not be written into v2 metadata.
+        assert "duplicate_at_runtime" not in meta
         assert meta[DATASET_META_DUPLICATION_FACTOR] == 5
         assert meta[DATASET_META_ORIGINAL_ITEMS] == 4
         assert meta[DATASET_META_TOTAL_ITEMS] == 20  # 4 rows × factor 5

backend/app/tests/services/evaluations/test_load_run_dataset_items.py

@@ -3,8 +3,8 @@
 Covers the v2 run-time-duplication slice of the three-metric SRD
 (docs/srd-three-metric-evaluation-verdict.md, FR-21/FR-22):
 
-- FR-21: a v2 dataset (null Langfuse id, run-time-duplication marker, factor N)
-  expands each original row ×N with unique ids at run time.
+- FR-21: a v2 dataset (null Langfuse id, stored factor N) expands each original
+  row ×N with unique ids at run time — the S3-backed shape alone drives it.
 - FR-22: a v1 dataset (Langfuse-backed) is read from Langfuse as-is, never
   re-multiplied, and its S3 CSV is not touched.
 
@@ -20,7 +20,6 @@
 from sqlmodel import Session
 
 from app.crud.evaluations.dataset import (
-    DATASET_META_DUPLICATE_AT_RUNTIME,
     DATASET_META_DUPLICATION_FACTOR,
     DATASET_META_ORIGINAL_ITEMS,
     DATASET_META_TOTAL_ITEMS,
@@ -52,17 +51,15 @@ def _make_v2_dataset(
     auth: TestAuthContext,
     original_items_count: int,
     duplication_factor: int,
-    duplicate_at_runtime: bool = True,
 ) -> EvaluationDataset:
-    """A Langfuse-free dataset: null langfuse id, S3 url, run-time-dup metadata."""
+    """A Langfuse-free dataset: null langfuse id, S3 url, stored-factor metadata."""
     return create_evaluation_dataset(
         session=db,
         name=f"v2_ds_{random_lower_string()}",
         dataset_metadata={
             DATASET_META_ORIGINAL_ITEMS: original_items_count,
             DATASET_META_TOTAL_ITEMS: original_items_count * duplication_factor,
             DATASET_META_DUPLICATION_FACTOR: duplication_factor,
-            DATASET_META_DUPLICATE_AT_RUNTIME: duplicate_at_runtime,
         },
         object_store_url="s3://bucket/datasets/v2.csv",
         langfuse_dataset_id=None,
@@ -118,16 +115,15 @@ def test_duplicates_of_a_row_share_question_id(
         assert sorted(groups) == [1, 2, 3]
         assert all(len(ids) == 4 for ids in groups.values())
 
-    def test_marker_absent_does_not_multiply(
+    def test_stored_factor_one_loads_rows_as_is(
         self, db: Session, user_api_key: TestAuthContext
     ) -> None:
-        """A null-langfuse dataset without the run-time-dup marker loads as-is."""
+        """An S3-backed dataset with stored factor 1 loads one item per row."""
         dataset = _make_v2_dataset(
             db=db,
             auth=user_api_key,
             original_items_count=5,
-            duplication_factor=5,
-            duplicate_at_runtime=False,
+            duplication_factor=1,
         )
 
         with (
@@ -190,31 +186,6 @@ def test_no_override_uses_stored_factor(
 
         assert len(items) == 40
 
-    def test_non_runtime_dataset_forces_one_despite_override(
-        self, db: Session, user_api_key: TestAuthContext
-    ) -> None:
-        """Defensive: a non-runtime dataset ignores the override and stays ×1."""
-        dataset = _make_v2_dataset(
-            db=db,
-            auth=user_api_key,
-            original_items_count=6,
-            duplication_factor=5,
-            duplicate_at_runtime=False,
-        )
-
-        with (
-            patch(f"{_FAST}.get_cloud_storage", return_value=MagicMock()),
-            patch(
-                f"{_FAST}.download_csv_from_object_store",
-                return_value=_csv_bytes(6),
-            ),
-        ):
-            items = _load_items_from_object_store(
-                session=db, dataset=dataset, duplication_factor=3
-            )
-
-        assert len(items) == 6
-
     def test_load_run_dataset_items_threads_override_to_object_store(
         self, db: Session, user_api_key: TestAuthContext
     ) -> None:

docs/wiki/modules/evaluations.md

@@ -23,7 +23,7 @@ All paths relative to `backend/app/`.
 | `batch_job` (BatchJob) | `models/batch_job.py` |
 | `evaluation_iteration_run` (EvaluationIterationRun) | `models/evaluation_iteration.py` |
 
-Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. `callback_url` (nullable): optional webhook set by the v2 run trigger; on terminal transition Kaapi POSTs a slim `APIResponse` snapshot to it (see Async). Not exposed on `EvaluationRunPublic` (no leak). `duplication_factor` (nullable): optional per-run override of the dataset's stored factor, set by the v2 run trigger for runtime-duplicated datasets only (else `422`); persisted on the run so the fan-out sizing, the chunk re-load (`execute_fast_evaluation_chunk`), and the ai_summary repetition math all use the same effective factor.
+Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. `callback_url` (nullable): optional webhook set by the v2 run trigger; on terminal transition Kaapi POSTs a slim `APIResponse` snapshot to it (see Async). Not exposed on `EvaluationRunPublic` (no leak). `duplication_factor` (nullable): optional per-run override of the dataset's stored factor, set by the v2 run trigger for S3-only (Langfuse-free) datasets only — rejected with `422` for Langfuse-backed datasets whose items are already physically duplicated; persisted on the run so the fan-out sizing, the chunk re-load (`execute_fast_evaluation_chunk`), and the ai_summary repetition math all use the same effective factor.
 v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Judge metrics score on an **integer 0–5 stepped scale** (the LLM returns integers 0–5; `crud/evaluations/judge.py::_parse_metric_score` enforces it) with English-only reasoning; scores are stored raw (0–5), so the API structure is unchanged but the value range is 0–5 (cosine on the v1 path stays 0–1). Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, 0–5 cutoffs 2/4 → 0–1 Needs Improvement, 2–3 Needs Refinement, 4–5 Good), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none.
 `score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, Anthropic call via the platform-owned `settings.ANTHROPIC_API_KEY` (same key as prompt improvement, no per-project credential), model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it).