feat(http): Add observability metrics tracking

backend/app/api/deps.py

@@ -15,7 +15,7 @@
 from app.core.config import settings
 from app.core.db import engine
 from app.core.security import api_key_manager
-from app.core.telemetry import set_request_log_context
+from app.core.telemetry import bind_sentry_user, set_request_log_context
 from app.crud.organization import validate_organization
 from app.crud.project import validate_project
 from app.models import (
@@ -45,12 +45,12 @@ def get_db() -> Generator[Session, None, None]:
 
 
 def _set_tenant_span_attributes(auth_context: AuthContext) -> None:
-    """Tag the active OTel span and log context with tenant info after auth.
+    """Tag the active OTel span, log context and Sentry scope with tenant info after auth.
 
     Sets org/project on:
     - OTel span   → Sentry traces filterable by tenant
     - log context → every log record in this request carries org_id/project_id
-    - Sentry scope → tags on all events for this request
+    - Sentry scope → tenant tags, plus the user binding that drives users-affected
     """
     span = trace.get_current_span()
     if span.is_recording():
@@ -60,10 +60,10 @@ def _set_tenant_span_attributes(auth_context: AuthContext) -> None:
         if auth_context.project:
             span.set_attribute("tenant.project_id", auth_context.project.id)
 
-    set_request_log_context(
-        org_id=auth_context.organization.id if auth_context.organization else None,
-        project_id=auth_context.project.id if auth_context.project else None,
-    )
+    org_id = auth_context.organization.id if auth_context.organization else None
+    project_id = auth_context.project.id if auth_context.project else None
+    set_request_log_context(org_id=org_id, project_id=project_id)
+    bind_sentry_user(user_id=auth_context.user.id, org_id=org_id, project_id=project_id)
 
 
 def _authenticate_with_jwt(session: Session, token: str) -> AuthContext:

backend/app/celery/celery_app.py

@@ -13,7 +13,12 @@
 
 from app.core.config import settings
 from app.core.logger import configure_logging
-from app.core.sentry_filters import before_send_transaction_filter
+from app.core.sentry_filters import (
+    before_send_error_filter,
+    before_send_log_filter,
+    before_send_transaction_filter,
+    genai_privacy_integrations,
+)
 
 logger = logging.getLogger(__name__)
 _telemetry_initialized = False
@@ -38,21 +43,39 @@ def _initialize_worker_observability() -> None:
         from sentry_sdk.integrations.logging import LoggingIntegration
         from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 
+        from app.core.telemetry import (
+            SENTRY_PROFILE_LIFECYCLE,
+            SENTRY_PROFILE_SESSION_SAMPLE_RATE,
+            resolve_sentry_release,
+        )
+
         sentry_sdk.init(
             dsn=str(settings.SENTRY_DSN),
             environment=settings.ENVIRONMENT,
-            release=settings.API_VERSION,
+            release=resolve_sentry_release(),
             instrumenter="otel",
-            traces_sample_rate=1.0,
+            traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE,
+            sample_rate=settings.SENTRY_ERROR_SAMPLE_RATE,
+            profile_session_sample_rate=SENTRY_PROFILE_SESSION_SAMPLE_RATE,
+            profile_lifecycle=SENTRY_PROFILE_LIFECYCLE,
+            send_default_pii=settings.SENTRY_SEND_DEFAULT_PII,
             enable_logs=True,
+            # Frame locals and task payloads hold prompts and completions verbatim.
+            include_local_variables=False,
+            max_request_body_size="never",
+            before_send=before_send_error_filter,
             before_send_transaction=before_send_transaction_filter,
+            before_send_log=before_send_log_filter,
             integrations=[
+                *genai_privacy_integrations(),
                 LoggingIntegration(
                     level=logging.INFO,
                     sentry_logs_level=logging.INFO,
                 ),
+                # propagate_traces=True links an API request to the task it
+                # enqueues as one trace; poll-loop re-enqueues opt out per-call.
                 CeleryIntegration(
-                    propagate_traces=False,
+                    propagate_traces=True,
                     monitor_beat_tasks=False,
                 ),
             ],

backend/app/celery/tasks/job_execution.py

@@ -24,6 +24,7 @@
 from app.celery.celery_app import celery_app
 from app.celery.utils import gevent_timeout
 from app.core.config import settings
+from app.core.telemetry import suppress_db_instrumentation
 
 if TYPE_CHECKING:
     from app.services.notifications.eval_completion import (
@@ -37,6 +38,9 @@
 # app/core/logger.py and app/celery/utils.py).
 DEFAULT_TRACE_ID = "N/A"
 
+# Start a fresh trace per poll cycle; otherwise one trace spans every re-enqueue.
+SENTRY_NO_PROPAGATE_HEADERS: dict[str, bool] = {"sentry-propagate-traces": False}
+
 
 def _set_trace(trace_id: str) -> None:
     correlation_id.set(trace_id)
@@ -89,16 +93,19 @@ def run_llm_job(self, project_id: int, job_id: str, trace_id: str, **kwargs):
     from app.services.llm.jobs import execute_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    # DB spans suppressed job-wide so LLM waterfalls stay clean (drops these queries
+    # from the Sentry Queries page too — accepted trade-off).
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)
@@ -107,16 +114,17 @@ def run_llm_chain_job(self, project_id: int, job_id: str, trace_id: str, **kwarg
     from app.services.llm.jobs import execute_chain_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_chain_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_chain_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)
@@ -125,16 +133,17 @@ def run_response_job(self, project_id: int, job_id: str, trace_id: str, **kwargs
     from app.services.response.jobs import execute_job
 
     _set_trace(trace_id)
-    return _run_with_otel_parent(
-        self,
-        lambda: execute_job(
-            project_id=project_id,
-            job_id=job_id,
-            task_id=current_task.request.id,
-            task_instance=self,
-            **kwargs,
-        ),
-    )
+    with suppress_db_instrumentation():
+        return _run_with_otel_parent(
+            self,
+            lambda: execute_job(
+                project_id=project_id,
+                job_id=job_id,
+                task_id=current_task.request.id,
+                task_instance=self,
+                **kwargs,
+            ),
+        )
 
 
 @celery_app.task(bind=True, queue="default", priority=9)
@@ -394,6 +403,7 @@ def run_assessment_api_batch(
                 "trace_id": trace_id,
             },
             countdown=POLL_COUNTDOWN_SECONDS,
+            headers=SENTRY_NO_PROPAGATE_HEADERS,
         )
     return result
 

backend/app/core/config.py

@@ -157,6 +157,12 @@ def AWS_S3_BUCKET(self) -> str:
     BACKEND_SERVICE_NAME: str = "kaapi-backend"
     CRON_SERVICE_NAME: str = "kaapi-cron"
 
+    # Defaults preserve current behavior; production .env needs no changes.
+    SENTRY_TRACES_SAMPLE_RATE: float = 1.0
+    SENTRY_RELEASE: str | None = None
+    SENTRY_SEND_DEFAULT_PII: bool = False
+    SENTRY_ERROR_SAMPLE_RATE: float = 1.0
+
     # Threshold Request Rate per minute
     THRESHOLD_LLM_CALL_RATE: int = 15
     THRESHOLD_COLLECTIONS_RATE: int = 3

backend/app/core/exception_handlers.py

@@ -1,6 +1,7 @@
 import re
 from collections import defaultdict
 
+import sentry_sdk
 from fastapi import FastAPI, Request, HTTPException
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
@@ -109,6 +110,8 @@ async def http_exception_handler(
 
     @app.exception_handler(Exception)
     async def generic_error_handler(request: Request, exc: Exception) -> JSONResponse:
+        # Capture within the active span so the Issue links to its trace.
+        sentry_sdk.capture_exception(exc)
         return JSONResponse(
             status_code=HTTP_500_INTERNAL_SERVER_ERROR,
             content=APIResponse.failure_response(

backend/app/core/finetune/evaluation.py

@@ -102,7 +102,7 @@ def load_labels_and_prompts(self) -> None:
             file_obj.close()
 
     def normalize_prediction(self, text: str) -> str:
-        logger.debug(f"[normalize_prediction] Normalizing prediction: {text}")
+        logger.debug(f"[normalize_prediction] Prediction length: {len(text or '')}")
         t = (text or "").strip().lower()
 
         if t in self.allowed_labels:

backend/app/core/middleware.py

@@ -19,6 +19,11 @@
     }
 )
 
+CRON_PATH_PREFIX: str = f"{settings.API_V1_STR}/cron/"
+
+# Excluded from traces only; logs/metrics and spans inside the handler still emit.
+TRACE_EXCLUDED_PATH_PREFIXES: frozenset[str] = frozenset({CRON_PATH_PREFIX})
+
 
 class StripTrailingSlashMiddleware:
     """
@@ -50,8 +55,44 @@ def _resolve_http_route(request: Request) -> str:
     return templated or "unmatched"
 
 
+def _emit_http_metrics(
+    *,
+    method: str,
+    http_route: str,
+    status: int,
+    duration_ms: float,
+    request_body_size: int = 0,
+) -> None:
+    """Emit HTTP traffic/latency/payload/error counters to Sentry. No-op if the SDK is inactive."""
+    try:
+        if not sentry_sdk.get_client().is_active():
+            return
+        attrs = {
+            "http.method": method,
+            "http.route": http_route,
+            "http.status_code": str(status),
+        }
+        sentry_sdk.metrics.count("http.server.request.count", 1, attributes=attrs)
+        sentry_sdk.metrics.distribution(
+            "http.server.request.duration",
+            duration_ms,
+            unit="millisecond",
+            attributes=attrs,
+        )
+        sentry_sdk.metrics.distribution(
+            "http.server.request.body.size",
+            request_body_size,
+            unit="byte",
+            attributes=attrs,
+        )
+        if status >= 400:
+            sentry_sdk.metrics.count("http.server.request.error", 1, attributes=attrs)
+    except Exception:
+        logger.debug("[_emit_http_metrics] Sentry metric emit failed")
+
+
 async def http_request_logger(request: Request, call_next) -> Response:
-    if request.url.path.startswith(f"{settings.API_V1_STR}/cron/"):
+    if request.url.path.startswith(CRON_PATH_PREFIX):
         with log_service_name(settings.CRON_SERVICE_NAME):
             return await _log_http_request(request, call_next)
 
@@ -73,6 +114,8 @@ async def _log_http_request(request: Request, call_next) -> Response:
     start_time = time.time()
     method = request.method
     raw_path = request.url.path
+    # Health/utility paths excluded so they don't skew platform traffic metrics.
+    metrics_enabled = raw_path not in SILENT_LOG_PATHS
     request_body_size = _resolve_request_body_size(request)
 
     span = trace.get_current_span()
@@ -91,6 +134,7 @@ async def _log_http_request(request: Request, call_next) -> Response:
     try:
         response = await call_next(request)
     except Exception:
+        duration_ms = (time.time() - start_time) * 1000
         status = 500
         http_route = _resolve_http_route(request)
         if span.is_recording():
@@ -101,6 +145,14 @@ async def _log_http_request(request: Request, call_next) -> Response:
             sentry_sdk.set_tag("http.route", http_route)
             sentry_sdk.set_tag("http.status_code", str(status))
             sentry_sdk.set_tag("http.response.status_code", str(status))
+        if metrics_enabled:
+            _emit_http_metrics(
+                method=method,
+                http_route=http_route,
+                status=status,
+                duration_ms=duration_ms,
+                request_body_size=request_body_size,
+            )
         logger.exception("Unhandled exception during request")
         raise
 
@@ -114,42 +166,23 @@ async def _log_http_request(request: Request, call_next) -> Response:
         span.set_attribute("http.response.status_code", status)
         span.set_attribute("http.request.duration_ms", round(duration_ms, 2))
 
-    if raw_path not in SILENT_LOG_PATHS:
+    if sentry_sdk.get_client().is_active():
+        sentry_sdk.set_tag("http.route", http_route)
+        sentry_sdk.set_tag("http.status_code", str(status))
+        sentry_sdk.set_tag("http.response.status_code", str(status))
+
+    if metrics_enabled:
         logger.info(
             f"[_log_http_request] {method} {raw_path} - {status} [{duration_ms:.2f}ms] "
             f"| request_body_size: {request_body_size}B "
             f"| correlation_id: {correlation_id.get()}"
         )
-
-    try:
-        if sentry_sdk.get_client().is_active():
-            sentry_sdk.set_tag("http.route", http_route)
-            sentry_sdk.set_tag("http.status_code", str(status))
-            sentry_sdk.set_tag("http.response.status_code", str(status))
-
-            attrs = {
-                "http.method": method,
-                "http.route": http_route,
-                "http.status_code": str(status),
-            }
-            sentry_sdk.metrics.count("http.server.request.count", 1, attributes=attrs)
-            sentry_sdk.metrics.distribution(
-                "http.server.request.body.size",
-                request_body_size,
-                unit="byte",
-                attributes=attrs,
-            )
-            sentry_sdk.metrics.distribution(
-                "http.server.request.duration",
-                duration_ms,
-                unit="millisecond",
-                attributes=attrs,
-            )
-            if status >= 400:
-                sentry_sdk.metrics.count(
-                    "http.server.request.error", 1, attributes=attrs
-                )
-    except Exception:
-        logger.debug("[http_request_logger] Sentry metric emit failed")
+        _emit_http_metrics(
+            method=method,
+            http_route=http_route,
+            status=status,
+            duration_ms=duration_ms,
+            request_body_size=request_body_size,
+        )
 
     return response