diff --git a/.env.example b/.env.example index 5a041fe26..e5b74445d 100644 --- a/.env.example +++ b/.env.example @@ -110,3 +110,5 @@ SMTP_PASSWORD= EMAILS_FROM_EMAIL= EMAILS_FROM_NAME=Kaapi FRONTEND_HOST= + +DELETE_ROLLING_WINDOW_HOURS= diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 9e11d033a..5100ac930 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -8,7 +8,9 @@ from app.api.permissions import Permission, require_permission from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations +from app.models.llm.response import LlmCallRedactionResult from app.services.job_monitoring import monitor_pending_jobs +from app.services.llm.retention import redact_aged_llm_calls from app.crud.stats import StatRow, get_daily_stats from app.services.stats import format_sections, post_to_discord @@ -23,15 +25,10 @@ "value": settings.CRON_INTERVAL_MINUTES, "unit": "minute", }, - # Timezone for the schedule (only affects crontab-style schedules). "timezone": "UTC", - # Grace period (minutes) before a late check-in is marked as missed. "checkin_margin": 2, - # Max runtime (minutes) before an in-progress run is marked as timed out. "max_runtime": 2 * settings.CRON_INTERVAL_MINUTES, - # Consecutive failures/missed/timeouts required to open a Sentry issue. "failure_issue_threshold": 2, - # Consecutive successful check-ins required to auto-resolve the issue. "recovery_threshold": 1, } @@ -59,6 +56,16 @@ } +LLM_CALL_RETENTION_CRON_MONITOR_CONFIG: MonitorConfig = { + "schedule": {"type": "crontab", "value": "0 9 * * *"}, + "timezone": "Asia/Kolkata", + "checkin_margin": 5, + "max_runtime": 30, + "failure_issue_threshold": 1, + "recovery_threshold": 1, +} + + @router.get( "/cron/evaluations", include_in_schema=False, @@ -158,6 +165,35 @@ def daily_stats_cron_job(session: SessionDep) -> dict[str, list[StatRow]]: raise +@router.get( + "/cron/llm-call-retention", + include_in_schema=False, + dependencies=[Depends(require_permission(Permission.SUPERUSER))], +) +@sentry_sdk.monitor( + monitor_slug="llm-call-retention-cron-job", + monitor_config=LLM_CALL_RETENTION_CRON_MONITOR_CONFIG, +) +def llm_call_retention_cron_job(session: SessionDep) -> LlmCallRedactionResult: + logger.info("[llm_call_retention_cron_job] Cron job invoked") + + try: + result = redact_aged_llm_calls(session=session) + logger.info( + f"[llm_call_retention_cron_job] Completed: " + f"rows_redacted={result.rows_redacted}, " + f"cutoff={result.cutoff}" + ) + return result + except Exception as e: + logger.error( + f"[llm_call_retention_cron_job] Error executing cron job: {e}", + exc_info=True, + ) + sentry_sdk.capture_exception(e) + raise + + @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index cc105fbec..10b652bbe 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -2,6 +2,7 @@ import os import secrets import warnings +from datetime import timedelta from typing import Any, Literal, Self from pydantic import ( @@ -196,10 +197,13 @@ def AWS_S3_BUCKET(self) -> str: EVAL_FAST_STALL_THRESHOLD_MINUTES: int = 15 PENDING_JOB_QUERY_TIMEOUT_MS: int = 1000 - # AI-assisted prompt improvement settings. - # See docs/srd-ai-prompt-improvement.md for the full design rationale. - # Platform-owned Anthropic key shared by every org/project for this feature, - # so prompt improvement works without per-project credentials. + DELETE_ROLLING_WINDOW_HOURS: int = 168 + + @computed_field # type: ignore[prop-decorator] + @property + def DELETE_ROLLING_WINDOW_TIMEDELTA(self) -> timedelta: + return timedelta(hours=self.DELETE_ROLLING_WINDOW_HOURS) + ANTHROPIC_API_KEY: str = "" PROMPT_IMPROVEMENT_MODEL: str = "claude-opus-4-8" diff --git a/backend/app/crud/llm.py b/backend/app/crud/llm.py index 44e605fb7..c807abc6a 100644 --- a/backend/app/crud/llm.py +++ b/backend/app/crud/llm.py @@ -1,9 +1,11 @@ import logging import base64 import json +from datetime import datetime from uuid import UUID from typing import Any, Literal +from sqlalchemy import text from sqlmodel import Session, select from app.core.util import now @@ -335,6 +337,38 @@ def get_llm_call_by_id( return session.exec(statement).first() +REDACTED_SENTINEL = "[redacted]" + +REDACT_LLM_CALLS_SQL = text( + """ + UPDATE llm_call + SET input = :redacted_sentinel, + content = CASE + WHEN jsonb_typeof(content) = 'object' + THEN jsonb_set(content, '{content,value}', to_jsonb(CAST(:redacted_sentinel AS text)), false) + ELSE content + END + WHERE updated_at <= :cutoff + AND (input <> :redacted_sentinel OR content -> 'content' ->> 'value' IS DISTINCT FROM :redacted_sentinel) + """ +) + + +def redact_llm_calls(*, session: Session, cutoff: datetime) -> int: + result = session.connection().execute( + REDACT_LLM_CALLS_SQL, + {"cutoff": cutoff, "redacted_sentinel": REDACTED_SENTINEL}, + ) + session.commit() + + logger.info( + f"[redact_llm_calls] Redacted rows | rows: {result.rowcount} | " + f"cutoff: {cutoff.isoformat()}" + ) + + return result.rowcount + + def get_llm_calls_by_job_id( session: Session, job_id: UUID, project_id: int ) -> list[LlmCall]: diff --git a/backend/app/models/llm/response.py b/backend/app/models/llm/response.py index 439a6adfc..afc6905ed 100644 --- a/backend/app/models/llm/response.py +++ b/backend/app/models/llm/response.py @@ -19,6 +19,13 @@ class Usage(SQLModel): reasoning_tokens: int | None = None +class LlmCallRedactionResult(SQLModel): + """Outcome of a single llm_call retention redaction run.""" + + rows_redacted: int + cutoff: datetime + + class TextOutput(SQLModel): type: Literal["text"] = "text" content: TextContent diff --git a/backend/app/services/llm/retention.py b/backend/app/services/llm/retention.py new file mode 100644 index 000000000..c7592cb71 --- /dev/null +++ b/backend/app/services/llm/retention.py @@ -0,0 +1,27 @@ +import logging + +from sqlmodel import Session + +from app.core.config import settings +from app.core.util import now +from app.crud.llm import redact_llm_calls +from app.models.llm.response import LlmCallRedactionResult + +logger = logging.getLogger(__name__) + + +def redact_aged_llm_calls(*, session: Session) -> LlmCallRedactionResult: + cutoff = now() - settings.DELETE_ROLLING_WINDOW_TIMEDELTA + + logger.info( + f"[redact_aged_llm_calls] Starting redaction | cutoff: {cutoff.isoformat()}" + ) + + rows_redacted = redact_llm_calls(session=session, cutoff=cutoff) + + logger.info( + f"[redact_aged_llm_calls] Completed | rows_redacted: {rows_redacted} | " + f"cutoff: {cutoff.isoformat()}" + ) + + return LlmCallRedactionResult(rows_redacted=rows_redacted, cutoff=cutoff) diff --git a/backend/app/tests/api/routes/test_cron.py b/backend/app/tests/api/routes/test_cron.py index 399ed6eee..9868eec77 100644 --- a/backend/app/tests/api/routes/test_cron.py +++ b/backend/app/tests/api/routes/test_cron.py @@ -1,3 +1,4 @@ +from datetime import datetime from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -5,6 +6,7 @@ from app.api.routes import cron from app.core.config import settings +from app.models.llm.response import LlmCallRedactionResult from app.tests.utils.auth import TestAuthContext @@ -326,6 +328,61 @@ def test_daily_stats_cron_job_captures_and_reraises_on_error() -> None: sentry.capture_exception.assert_called_once() +def test_llm_call_retention_cron_job_success( + client: TestClient, + superuser_api_key: TestAuthContext, +) -> None: + """Returns the redaction summary produced by the retention service.""" + result = LlmCallRedactionResult( + rows_redacted=4137, + cutoff=datetime(2026, 2, 22, 12, 0, 0), + ) + with patch( + "app.api.routes.cron.redact_aged_llm_calls", + return_value=result, + ) as redact: + response = client.get( + f"{settings.API_V1_STR}/cron/llm-call-retention", + headers={"X-API-KEY": superuser_api_key.key}, + ) + + assert response.status_code == 200 + assert response.json() == { + "rows_redacted": 4137, + "cutoff": "2026-02-22T12:00:00", + } + redact.assert_called_once() + + +def test_llm_call_retention_cron_job_requires_superuser( + client: TestClient, + user_api_key: TestAuthContext, +) -> None: + """Non-superuser cannot access the llm_call retention cron endpoint.""" + response = client.get( + f"{settings.API_V1_STR}/cron/llm-call-retention", + headers={"X-API-KEY": user_api_key.key}, + ) + + assert response.status_code == 403 + assert "Insufficient permissions" in response.json()["error"] + + +def test_llm_call_retention_cron_job_captures_and_reraises_on_error() -> None: + """On failure the job reports to Sentry and re-raises.""" + with ( + patch( + "app.api.routes.cron.redact_aged_llm_calls", + side_effect=RuntimeError("boom"), + ), + patch("app.api.routes.cron.sentry_sdk") as sentry, + ): + with pytest.raises(RuntimeError): + cron.llm_call_retention_cron_job(session=MagicMock()) + + sentry.capture_exception.assert_called_once() + + def test_evaluation_cron_job_not_in_schema( client: TestClient, ) -> None: @@ -340,6 +397,7 @@ def test_evaluation_cron_job_not_in_schema( assert f"{settings.API_V1_STR}/cron/evaluations" not in paths assert f"{settings.API_V1_STR}/cron/pending-jobs" not in paths assert f"{settings.API_V1_STR}/cron/daily-stats" not in paths + assert f"{settings.API_V1_STR}/cron/llm-call-retention" not in paths def test_cron_intervals_match_to_prevent_sentry_monitor_drift() -> None: diff --git a/backend/app/tests/crud/test_llm_retention.py b/backend/app/tests/crud/test_llm_retention.py new file mode 100644 index 000000000..f8888c9ef --- /dev/null +++ b/backend/app/tests/crud/test_llm_retention.py @@ -0,0 +1,149 @@ +from datetime import datetime, timedelta + +import pytest +from sqlalchemy import text +from sqlmodel import Session + +from app.models import Job +from app.crud.llm import REDACTED_SENTINEL, redact_llm_calls +from app.models.llm import LlmCall +from app.tests.utils.llm import create_aged_llm_call, create_llm_job + +CUTOFF = datetime(2026, 1, 8, 0, 0, 0) +AGED = CUTOFF - timedelta(days=1) +RECENT = CUTOFF + timedelta(minutes=1) + +TEXT_CONTENT = {"type": "text", "content": {"format": "text", "value": "hello"}} + + +@pytest.fixture +def job(db: Session) -> Job: + # The statement targets llm_call_2, a copy table that only exists in the + # deployed databases; an auto-updatable view over llm_call gives the tests + # that name while keeping the real column types and the ORM factories. + db.exec(text("DROP VIEW IF EXISTS llm_call_2")) + # Other llm_call rows (seed data, sibling fixtures) would also match the + # cutoff and skew the rowcount assertions, so start from an empty table. + db.exec(text("DELETE FROM llm_call")) + db.exec(text("CREATE VIEW llm_call_2 AS SELECT * FROM llm_call")) + db.commit() + yield create_llm_job(db) + db.exec(text("DROP VIEW IF EXISTS llm_call_2")) + db.commit() + + +def _reload(db: Session, llm_call: LlmCall) -> LlmCall: + db.expire_all() + return db.get(LlmCall, llm_call.id) + + +def test_redacts_input_and_content_value_only(db: Session, job: Job) -> None: + llm_call = create_aged_llm_call( + db, + updated_at=AGED, + job_id=job.id, + input="my social security number is 123-45-6789", + content=TEXT_CONTENT, + ) + + redacted = redact_llm_calls(session=db, cutoff=CUTOFF) + + assert redacted == 1 + row = _reload(db, llm_call) + assert row.input == REDACTED_SENTINEL + assert row.content["content"]["value"] == REDACTED_SENTINEL + assert row.content["content"]["format"] == "text" + assert row.content["type"] == "text" + + +def test_leaves_rows_newer_than_cutoff_untouched(db: Session, job: Job) -> None: + llm_call = create_aged_llm_call( + db, + updated_at=RECENT, + job_id=job.id, + input="still within the retention window", + content=TEXT_CONTENT, + ) + + redacted = redact_llm_calls(session=db, cutoff=CUTOFF) + + assert redacted == 0 + row = _reload(db, llm_call) + assert row.input == "still within the retention window" + assert row.content["content"]["value"] == "hello" + + +def test_row_exactly_at_cutoff_is_redacted(db: Session, job: Job) -> None: + llm_call = create_aged_llm_call( + db, + updated_at=CUTOFF, + job_id=job.id, + content=TEXT_CONTENT, + ) + + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 1 + assert _reload(db, llm_call).input == REDACTED_SENTINEL + + +def test_second_run_skips_already_redacted_rows(db: Session, job: Job) -> None: + create_aged_llm_call(db, updated_at=AGED, job_id=job.id, content=TEXT_CONTENT) + + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 1 + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 0 + + +def test_row_with_null_content_is_redacted_without_error(db: Session, job: Job) -> None: + llm_call = create_aged_llm_call( + db, + updated_at=AGED, + job_id=job.id, + input="no response was ever recorded", + content=None, + ) + # A call that never got a response stores the JSONB scalar 'null', not SQL + # NULL; jsonb_set errors on scalars, so this must take a guarded path. + assert ( + db.exec( + text( + "SELECT jsonb_typeof(content) FROM llm_call WHERE id = :id" + ).bindparams(id=llm_call.id) + ).scalar_one() + == "null" + ) + + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 1 + row = _reload(db, llm_call) + assert row.input == REDACTED_SENTINEL + assert row.content is None + + +def test_returns_zero_when_no_rows_match(db: Session, job: Job) -> None: + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 0 + + +def test_redacts_only_aged_rows_in_a_mixed_table(db: Session, job: Job) -> None: + aged = create_aged_llm_call( + db, updated_at=AGED, job_id=job.id, input="aged", content=TEXT_CONTENT + ) + recent = create_aged_llm_call( + db, updated_at=RECENT, job_id=job.id, input="recent", content=TEXT_CONTENT + ) + + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 1 + assert _reload(db, aged).input == REDACTED_SENTINEL + assert _reload(db, recent).input == "recent" + + +def test_redacts_content_value_of_row_whose_input_is_already_sentinel( + db: Session, job: Job +) -> None: + llm_call = create_aged_llm_call( + db, + updated_at=AGED, + job_id=job.id, + input=REDACTED_SENTINEL, + content={"type": "audio", "content": {"format": "uri", "value": "s3://a.wav"}}, + ) + + assert redact_llm_calls(session=db, cutoff=CUTOFF) == 1 + assert _reload(db, llm_call).content["content"]["value"] == REDACTED_SENTINEL diff --git a/backend/app/tests/services/llm/test_retention.py b/backend/app/tests/services/llm/test_retention.py new file mode 100644 index 000000000..8d5881174 --- /dev/null +++ b/backend/app/tests/services/llm/test_retention.py @@ -0,0 +1,75 @@ +from datetime import datetime, timedelta + +import pytest +from sqlmodel import Session + +from app.core.config import settings +from app.services.llm import retention +from app.services.llm.retention import redact_aged_llm_calls + + +def _stub_redact( + monkeypatch: pytest.MonkeyPatch, calls: list[dict], returns: int = 0 +) -> None: + def fake_redact(**kwargs) -> int: + calls.append(kwargs) + return returns + + monkeypatch.setattr(retention, "redact_llm_calls", fake_redact) + + +@pytest.fixture +def redact_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]: + calls: list[dict] = [] + _stub_redact(monkeypatch, calls) + return calls + + +def test_reports_rows_redacted_from_a_single_crud_call( + db: Session, monkeypatch: pytest.MonkeyPatch +) -> None: + calls: list[dict] = [] + _stub_redact(monkeypatch, calls, returns=4137) + + result = redact_aged_llm_calls(session=db) + + assert result.rows_redacted == 4137 + assert len(calls) == 1 + + +def test_no_matching_rows(db: Session, redact_calls: list[dict]) -> None: + result = redact_aged_llm_calls(session=db) + + assert result.rows_redacted == 0 + assert len(redact_calls) == 1 + + +def test_cutoff_is_naive_utc(db: Session, redact_calls: list[dict]) -> None: + # llm_call.updated_at is a naive TIMESTAMP; a tz-aware cutoff would raise on compare. + result = redact_aged_llm_calls(session=db) + + cutoff = redact_calls[0]["cutoff"] + assert cutoff.tzinfo is None + assert result.cutoff == cutoff + + +def test_cutoff_trails_now_by_the_configured_rolling_window( + db: Session, redact_calls: list[dict], monkeypatch: pytest.MonkeyPatch +) -> None: + frozen = datetime(2026, 3, 1, 12, 0, 0) + monkeypatch.setattr(retention, "now", lambda: frozen) + monkeypatch.setattr(settings, "DELETE_ROLLING_WINDOW_HOURS", 168) + + redact_aged_llm_calls(session=db) + + assert redact_calls[0]["cutoff"] == datetime(2026, 2, 22, 12, 0, 0) + + +def test_passes_session_to_crud(db: Session, redact_calls: list[dict]) -> None: + redact_aged_llm_calls(session=db) + + assert redact_calls[0]["session"] is db + + +def test_default_rolling_window_is_one_week() -> None: + assert settings.DELETE_ROLLING_WINDOW_TIMEDELTA == timedelta(days=7) diff --git a/backend/app/tests/utils/llm.py b/backend/app/tests/utils/llm.py index a60059b87..da84528c2 100644 --- a/backend/app/tests/utils/llm.py +++ b/backend/app/tests/utils/llm.py @@ -1,8 +1,13 @@ +from datetime import datetime +from typing import Any +from uuid import UUID + from sqlmodel import Session from app.crud import JobCrud from app.crud.llm import create_llm_call, update_llm_call_response from app.models import JobType, Job +from app.models.llm import LlmCall from app.models.llm.response import LLMCallResponse from app.models.llm.request import ( ConfigBlob, @@ -10,10 +15,45 @@ QueryParams, build_kaapi_completion_config, ) -from app.tests.utils.utils import get_project +from app.tests.utils.utils import get_project, get_organization from app.models.llm import LLMCallRequest +def create_aged_llm_call( + db: Session, + *, + updated_at: datetime, + job_id: UUID, + input: str = "what is the capital of France?", + content: dict[str, Any] | None = None, +) -> LlmCall: + """Persist an LlmCall with an explicit `updated_at`, for retention tests.""" + project = get_project(db, "Dalgo") + organization = get_organization(db) + + llm_call = LlmCall( + job_id=job_id, + project_id=project.id, + organization_id=organization.id, + input=input, + input_type="text", + output_type="text", + provider="openai", + model="gpt-4o", + content=content, + ) + db.add(llm_call) + db.commit() + + # updated_at has a server/model default, so it is overwritten after insert. + llm_call.updated_at = updated_at + db.add(llm_call) + db.commit() + db.refresh(llm_call) + + return llm_call + + def create_llm_job(db: Session) -> Job: """Create a persisted LLM_API job for use in tests.""" project = get_project(db, "Dalgo") diff --git a/scripts/python/invoke-cron.py b/scripts/python/invoke-cron.py index e589c65aa..2156b00ad 100644 --- a/scripts/python/invoke-cron.py +++ b/scripts/python/invoke-cron.py @@ -19,6 +19,7 @@ "/api/v1/cron/evaluations", "/api/v1/cron/pending-jobs", "/api/v1/cron/daily-stats", + "/api/v1/cron/llm-call-retention", ] REQUEST_TIMEOUT = 30 # Timeout for requests in seconds