diff --git a/backend/app/crud/evaluations/dataset.py b/backend/app/crud/evaluations/dataset.py index f38de95bb..81407cc00 100644 --- a/backend/app/crud/evaluations/dataset.py +++ b/backend/app/crud/evaluations/dataset.py @@ -33,10 +33,6 @@ DATASET_META_ORIGINAL_ITEMS = "original_items_count" DATASET_META_TOTAL_ITEMS = "total_items_count" DATASET_META_DUPLICATION_FACTOR = "duplication_factor" -# v2 marker: the stored CSV holds only the original rows and duplication is applied -# at run time. Absent/false means the S3 data is already physically duplicated (v1), -# so the run reads it as-is and must not multiply again. -DATASET_META_DUPLICATE_AT_RUNTIME = "duplicate_at_runtime" def create_evaluation_dataset( diff --git a/backend/app/services/evaluations/dataset.py b/backend/app/services/evaluations/dataset.py index bba473c17..0872b39b7 100644 --- a/backend/app/services/evaluations/dataset.py +++ b/backend/app/services/evaluations/dataset.py @@ -12,7 +12,6 @@ upload_dataset_to_langfuse, ) from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, DATASET_META_ORIGINAL_ITEMS, DATASET_META_TOTAL_ITEMS, @@ -162,9 +161,6 @@ def upload_dataset( DATASET_META_TOTAL_ITEMS: total_items_count, DATASET_META_DUPLICATION_FACTOR: duplication_factor, } - if not use_langfuse: - # The stored CSV holds only the original rows; the run expands them. - metadata[DATASET_META_DUPLICATE_AT_RUNTIME] = True dataset = create_evaluation_dataset( session=session, diff --git a/backend/app/services/evaluations/fast.py b/backend/app/services/evaluations/fast.py index 17e671db1..552749fe7 100644 --- a/backend/app/services/evaluations/fast.py +++ b/backend/app/services/evaluations/fast.py @@ -29,7 +29,6 @@ from app.crud.evaluations.batch import fetch_dataset_items from app.crud.evaluations.core import update_evaluation_run from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, download_csv_from_object_store, ) @@ -72,14 +71,13 @@ def load_run_dataset_items( - Langfuse-backed dataset (v1): items are already physically duplicated in Langfuse; read as-is via `fetch_dataset_items`, never re-multiplied. - - S3-only dataset (v2, `langfuse_dataset_id` NULL): download the original-items - CSV and, when the dataset is marked for run-time duplication, expand each row - ×duplication_factor with a unique item id per copy. + - S3-only dataset (`langfuse_dataset_id` NULL): download the original-items CSV + and expand each original row ×duplication_factor with a unique item id per copy. Both the fan-out sizing and the per-chunk load call this, so they agree on the same expanded item set. `duplication_factor`, when set, overrides the dataset's - stored factor for runtime-duplicated datasets only (see - `_load_items_from_object_store`); it never applies to a Langfuse dataset. + stored factor (see `_load_items_from_object_store`); it never applies to a + Langfuse dataset. """ if dataset.langfuse_dataset_id: if langfuse is None: @@ -102,10 +100,10 @@ def _load_items_from_object_store( ) -> list[dict[str, Any]]: """Parse the dataset's original-items CSV from S3 into fast-pipeline items. - When the dataset is marked for run-time duplication (v2), each original row is - emitted `duplication_factor` times with a distinct item id (`item_{row}_{dup}`) - so per-row score keys stay unique. A v1 dataset's S3 CSV is already physically - duplicated, so it loads as-is (factor forced to 1).""" + This loader is only reached for S3-only datasets (`langfuse_dataset_id` NULL), + whose stored CSV always holds the original (un-duplicated) rows. Each original + row is emitted `duplication_factor` times with a distinct item id + (`item_{row}_{dup}`) so per-row score keys stay unique.""" if not dataset.object_store_url: raise ValueError(f"Dataset {dataset.id} has no object-store CSV to load") @@ -116,13 +114,12 @@ def _load_items_from_object_store( original_items = parse_csv_items(csv_content) metadata = dataset.dataset_metadata or {} - duplicate_at_runtime = bool(metadata.get(DATASET_META_DUPLICATE_AT_RUNTIME, False)) effective_factor = ( duplication_factor if duplication_factor is not None else int(metadata.get(DATASET_META_DUPLICATION_FACTOR, 1)) ) - duplication_factor = max(1, effective_factor) if duplicate_at_runtime else 1 + duplication_factor = max(1, effective_factor) items: list[dict[str, Any]] = [] for row_idx, item in enumerate(original_items): @@ -169,8 +166,8 @@ def validate_fast_evaluation_inputs( 3. Dataset's original_items_count <= EVAL_FAST_MAX_UNIQUE_ROWS. `duplication_factor`, when provided, overrides the dataset's stored factor for - this run only and is supported for runtime-duplicated (v2) datasets exclusively - (rejected with 422 otherwise). + this run only and is supported for S3-only datasets exclusively; it is rejected + with 422 for Langfuse-backed datasets (whose items come pre-duplicated). """ # 1. Dataset must exist (Langfuse id required for v1 runs only; see below). dataset = get_dataset_by_id( @@ -198,14 +195,12 @@ def validate_fast_evaluation_inputs( ), ) - if duplication_factor is not None and not (dataset.dataset_metadata or {}).get( - DATASET_META_DUPLICATE_AT_RUNTIME - ): + if duplication_factor is not None and dataset.langfuse_dataset_id: raise HTTPException( status_code=422, detail=( - f"{ERR_DUPLICATION_FACTOR_NOT_SUPPORTED}: this dataset is not " - "runtime-duplicated; re-upload to change its factor" + f"{ERR_DUPLICATION_FACTOR_NOT_SUPPORTED}: this dataset is " + "Langfuse-backed and pre-duplicated; re-upload to change its factor" ), ) @@ -291,8 +286,8 @@ def validate_and_start_fast_evaluation( it stays NULL and no webhook fires. `duplication_factor`, when provided, overrides the dataset's stored factor for - this run only and is supported for runtime-duplicated (v2) datasets exclusively - (rejected with 422 otherwise). + this run only and is supported for S3-only datasets exclusively; it is rejected + with 422 for Langfuse-backed datasets (whose items come pre-duplicated). """ logger.info( f"[validate_and_start_fast_evaluation] Starting fast eval | " diff --git a/backend/app/tests/api/routes/test_evaluation_dataset_v2.py b/backend/app/tests/api/routes/test_evaluation_dataset_v2.py index 54ab0e643..f4e9727a8 100644 --- a/backend/app/tests/api/routes/test_evaluation_dataset_v2.py +++ b/backend/app/tests/api/routes/test_evaluation_dataset_v2.py @@ -5,8 +5,8 @@ - FR-19: 200, row created with null langfuse id, CSV stored in S3, Langfuse never called. -- FR-20: response + persisted metadata carry the run-time-duplication marker and - original/total item counts. +- FR-20: response + persisted metadata carry the original/total item counts and + the stored duplication factor. Object storage and Langfuse are the external boundaries and are mocked; the dataset row lands in the real (transactional) DB. @@ -20,7 +20,6 @@ from app.core.config import settings from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, DATASET_META_ORIGINAL_ITEMS, DATASET_META_TOTAL_ITEMS, @@ -45,7 +44,7 @@ def test_upload_creates_langfuse_free_dataset( user_api_key_header: dict[str, str], db: Session, ) -> None: - """FR-19/FR-20: 200, null langfuse id, S3 stored, run-time-dup metadata.""" + """FR-19/FR-20: 200, null langfuse id, S3 stored, factor/count metadata.""" name = f"v2-route-{random_lower_string()}" with ( patch(f"{_DATASET}.get_cloud_storage", return_value=MagicMock()), @@ -79,7 +78,6 @@ def test_upload_creates_langfuse_free_dataset( assert persisted is not None assert persisted.langfuse_dataset_id is None meta = persisted.dataset_metadata - assert meta[DATASET_META_DUPLICATE_AT_RUNTIME] is True assert meta[DATASET_META_DUPLICATION_FACTOR] == 5 assert meta[DATASET_META_ORIGINAL_ITEMS] == 3 assert meta[DATASET_META_TOTAL_ITEMS] == 15 diff --git a/backend/app/tests/api/routes/test_evaluation_v2.py b/backend/app/tests/api/routes/test_evaluation_v2.py index 038b90816..14930d27f 100644 --- a/backend/app/tests/api/routes/test_evaluation_v2.py +++ b/backend/app/tests/api/routes/test_evaluation_v2.py @@ -18,7 +18,6 @@ from app.core.config import settings from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, DATASET_META_ORIGINAL_ITEMS, DATASET_META_TOTAL_ITEMS, @@ -49,7 +48,7 @@ def _make_dataset(*, db: Session, user_api_key: TestAuthContext) -> EvaluationDa def _make_runtime_dup_dataset( *, db: Session, user_api_key: TestAuthContext, duplication_factor: int = 5 ) -> EvaluationDataset: - """A v2 runtime-duplicated dataset: null Langfuse id, S3 url, runtime marker.""" + """A v2 runtime-duplicated dataset: null Langfuse id, S3 url, stored factor.""" original = 3 return create_evaluation_dataset( session=db, @@ -58,7 +57,6 @@ def _make_runtime_dup_dataset( DATASET_META_ORIGINAL_ITEMS: original, DATASET_META_TOTAL_ITEMS: original * duplication_factor, DATASET_META_DUPLICATION_FACTOR: duplication_factor, - DATASET_META_DUPLICATE_AT_RUNTIME: True, }, object_store_url="s3://bucket/datasets/v2.csv", langfuse_dataset_id=None, diff --git a/backend/app/tests/services/evaluations/test_dataset_v2.py b/backend/app/tests/services/evaluations/test_dataset_v2.py index 55e775a1f..8b4ecc92d 100644 --- a/backend/app/tests/services/evaluations/test_dataset_v2.py +++ b/backend/app/tests/services/evaluations/test_dataset_v2.py @@ -6,7 +6,7 @@ - FR-19: creates the `evaluation_dataset` row with `langfuse_dataset_id` null and never touches the Langfuse client. - FR-20: stores only the original rows (no physical duplication) and records the - run-time-duplication metadata. + original/total counts and stored duplication factor. Object storage is the external boundary and is mocked; the dataset row lands in the real (transactional) DB. @@ -19,7 +19,6 @@ from sqlmodel import Session from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, DATASET_META_ORIGINAL_ITEMS, DATASET_META_TOTAL_ITEMS, @@ -68,10 +67,10 @@ def test_creates_langfuse_free_row_without_calling_langfuse( assert persisted.langfuse_dataset_id is None assert persisted.object_store_url == "s3://bucket/datasets/v2.csv" - def test_stores_original_rows_and_runtime_dup_metadata( + def test_stores_original_rows_and_factor_metadata( self, db: Session, user_api_key: TestAuthContext ) -> None: - """FR-20: original CSV stored verbatim; metadata records run-time dup.""" + """FR-20: original CSV stored verbatim; metadata records the stored factor.""" name = f"v2-meta-{random_lower_string()}" with ( patch(f"{_DATASET}.get_cloud_storage", return_value=MagicMock()), @@ -97,7 +96,8 @@ def test_stores_original_rows_and_runtime_dup_metadata( assert mock_upload.call_args.kwargs["csv_content"] == _CSV meta = dataset.dataset_metadata - assert meta[DATASET_META_DUPLICATE_AT_RUNTIME] is True + # The removed run-time-duplication flag must not be written into v2 metadata. + assert "duplicate_at_runtime" not in meta assert meta[DATASET_META_DUPLICATION_FACTOR] == 5 assert meta[DATASET_META_ORIGINAL_ITEMS] == 4 assert meta[DATASET_META_TOTAL_ITEMS] == 20 # 4 rows × factor 5 diff --git a/backend/app/tests/services/evaluations/test_load_run_dataset_items.py b/backend/app/tests/services/evaluations/test_load_run_dataset_items.py index 4becbbe0a..8430569ad 100644 --- a/backend/app/tests/services/evaluations/test_load_run_dataset_items.py +++ b/backend/app/tests/services/evaluations/test_load_run_dataset_items.py @@ -3,8 +3,8 @@ Covers the v2 run-time-duplication slice of the three-metric SRD (docs/srd-three-metric-evaluation-verdict.md, FR-21/FR-22): -- FR-21: a v2 dataset (null Langfuse id, run-time-duplication marker, factor N) - expands each original row ×N with unique ids at run time. +- FR-21: a v2 dataset (null Langfuse id, stored factor N) expands each original + row ×N with unique ids at run time — the S3-backed shape alone drives it. - FR-22: a v1 dataset (Langfuse-backed) is read from Langfuse as-is, never re-multiplied, and its S3 CSV is not touched. @@ -20,7 +20,6 @@ from sqlmodel import Session from app.crud.evaluations.dataset import ( - DATASET_META_DUPLICATE_AT_RUNTIME, DATASET_META_DUPLICATION_FACTOR, DATASET_META_ORIGINAL_ITEMS, DATASET_META_TOTAL_ITEMS, @@ -52,9 +51,8 @@ def _make_v2_dataset( auth: TestAuthContext, original_items_count: int, duplication_factor: int, - duplicate_at_runtime: bool = True, ) -> EvaluationDataset: - """A Langfuse-free dataset: null langfuse id, S3 url, run-time-dup metadata.""" + """A Langfuse-free dataset: null langfuse id, S3 url, stored-factor metadata.""" return create_evaluation_dataset( session=db, name=f"v2_ds_{random_lower_string()}", @@ -62,7 +60,6 @@ def _make_v2_dataset( DATASET_META_ORIGINAL_ITEMS: original_items_count, DATASET_META_TOTAL_ITEMS: original_items_count * duplication_factor, DATASET_META_DUPLICATION_FACTOR: duplication_factor, - DATASET_META_DUPLICATE_AT_RUNTIME: duplicate_at_runtime, }, object_store_url="s3://bucket/datasets/v2.csv", langfuse_dataset_id=None, @@ -118,16 +115,15 @@ def test_duplicates_of_a_row_share_question_id( assert sorted(groups) == [1, 2, 3] assert all(len(ids) == 4 for ids in groups.values()) - def test_marker_absent_does_not_multiply( + def test_stored_factor_one_loads_rows_as_is( self, db: Session, user_api_key: TestAuthContext ) -> None: - """A null-langfuse dataset without the run-time-dup marker loads as-is.""" + """An S3-backed dataset with stored factor 1 loads one item per row.""" dataset = _make_v2_dataset( db=db, auth=user_api_key, original_items_count=5, - duplication_factor=5, - duplicate_at_runtime=False, + duplication_factor=1, ) with ( @@ -190,31 +186,6 @@ def test_no_override_uses_stored_factor( assert len(items) == 40 - def test_non_runtime_dataset_forces_one_despite_override( - self, db: Session, user_api_key: TestAuthContext - ) -> None: - """Defensive: a non-runtime dataset ignores the override and stays ×1.""" - dataset = _make_v2_dataset( - db=db, - auth=user_api_key, - original_items_count=6, - duplication_factor=5, - duplicate_at_runtime=False, - ) - - with ( - patch(f"{_FAST}.get_cloud_storage", return_value=MagicMock()), - patch( - f"{_FAST}.download_csv_from_object_store", - return_value=_csv_bytes(6), - ), - ): - items = _load_items_from_object_store( - session=db, dataset=dataset, duplication_factor=3 - ) - - assert len(items) == 6 - def test_load_run_dataset_items_threads_override_to_object_store( self, db: Session, user_api_key: TestAuthContext ) -> None: diff --git a/docs/wiki/modules/evaluations.md b/docs/wiki/modules/evaluations.md index 8e27bf992..8ae4d7169 100644 --- a/docs/wiki/modules/evaluations.md +++ b/docs/wiki/modules/evaluations.md @@ -23,7 +23,7 @@ All paths relative to `backend/app/`. | `batch_job` (BatchJob) | `models/batch_job.py` | | `evaluation_iteration_run` (EvaluationIterationRun) | `models/evaluation_iteration.py` | -Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. `callback_url` (nullable): optional webhook set by the v2 run trigger; on terminal transition Kaapi POSTs a slim `APIResponse` snapshot to it (see Async). Not exposed on `EvaluationRunPublic` (no leak). `duplication_factor` (nullable): optional per-run override of the dataset's stored factor, set by the v2 run trigger for runtime-duplicated datasets only (else `422`); persisted on the run so the fan-out sizing, the chunk re-load (`execute_fast_evaluation_chunk`), and the ai_summary repetition math all use the same effective factor. +Key `EvaluationRun` JSONB fields: `score` (per-trace `scores` + `summary_scores`), `per_item_scores`, `cost` (per-stage), `unscoreable`. References `config_id`, `batch_job_id`. `callback_url` (nullable): optional webhook set by the v2 run trigger; on terminal transition Kaapi POSTs a slim `APIResponse` snapshot to it (see Async). Not exposed on `EvaluationRunPublic` (no leak). `duplication_factor` (nullable): optional per-run override of the dataset's stored factor, set by the v2 run trigger for S3-only (Langfuse-free) datasets only — rejected with `422` for Langfuse-backed datasets whose items are already physically duplicated; persisted on the run so the fan-out sizing, the chunk re-load (`execute_fast_evaluation_chunk`), and the ai_summary repetition math all use the same effective factor. v2 judge field on `EvaluationRun`: `is_judge_run` (bool marker gating native judging + Langfuse skip). All judge metrics are trace-only — per-row score + reasoning live in the `score_trace_url` trace unit (the native source of truth); there is no per-metric backup column. Judging is system-config only — always the fallback model (`gpt-5-mini`) + built-in prompts, no per-run config. Judge metrics score on an **integer 0–5 stepped scale** (the LLM returns integers 0–5; `crud/evaluations/judge.py::_parse_metric_score` enforces it) with English-only reasoning; scores are stored raw (0–5), so the API structure is unchanged but the value range is 0–5 (cosine on the v1 path stays 0–1). Each numeric judge-metric trace score also carries a `verdict` band (`crud/evaluations/score.py`: `VerdictEnum` + `verdict_from_score`, 0–5 cutoffs 2/4 → 0–1 Needs Improvement, 2–3 Needs Refinement, 4–5 Good), set in the `crud/evaluations/fast.py` trace-build loop; cosine and unscoreable/`N/A` entries carry none. `score.overall` (`OverallSummary`, `crud/evaluations/score.py`): run-level weighted rollup for judge runs, computed by `compute_overall_summary` from each metric's `avg` + `METRIC_REGISTRY` weight (renormalized over metrics that actually scored ≥1 row) — `overall_score`, `verdict`, per-metric `breakdown` (score/weight/delta/verdict), plus `ai_summary` (best-effort natural-language note, `crud/evaluations/summary.py`, Anthropic call via the platform-owned `settings.ANTHROPIC_API_KEY` (same key as prompt improvement, no per-project credential), model = `settings.EVAL_SUMMARY_MODEL`; `None` on any failure, never fails the run). Persisted on `run_fast_evaluation`'s final `EvaluationRun.score` write and re-attached verbatim by `services/evaluations/evaluation.py::get_evaluation_with_scores` on every cache/resync path (trace merging never recomputes it).