From a84caa73755f02c70bd0d296e73d22e7fd5a3833 Mon Sep 17 00:00:00 2001 From: Harry Randall Date: Tue, 22 Sep 2026 13:34:47 +1000 Subject: [PATCH] feat: give developers the sync diagnostics the record pages hide Branch 04 removed the imports screen and with it every way to see why a sync failed. /admin/operations/catalogue restores that engineering visibility without putting pipeline concepts back in front of administrators: syncs and discovery checks with their statuses, attempts, leases, durations, models and costs, and one sync opening to its stages, its source document, its extractions and every stored artefact from raw HTML to the projected content. The artefact viewer, its viewport and the artefact route are the ones that existed before, recovered rather than rewritten. Operations sits behind imports.manage while authoring sits behind catalogue.write, so an administrator who edits content does not automatically read model prompts, responses and costs. The two names for the first permission became one, canManageCatalogueOperations. A record links out to its diagnostics and, when a sync fails, to the technical detail behind the failure, rather than growing a fifth tab. The inbox now talks about records: COMP2700 sync failed, or COMP2700 has 3 ANU changes to review, for whoever asked. A sync that found nothing, and a review with nothing to decide, say nothing. --- .../catalogue/[[...section]]/page.tsx | 47 ++ apps/web/app/admin/page.tsx | 4 +- .../api/admin/catalogue-directory/route.ts | 4 +- .../artifacts/[artifactId]/route.ts | 49 ++ .../app/api/admin/catalogue-syncs/route.ts | 6 +- apps/web/lib/admin/settings-actions.ts | 10 +- apps/web/lib/assistant/model-actions.ts | 4 +- apps/web/lib/auth/viewer.ts | 12 +- apps/web/lib/catalogue-sync/sync-service.ts | 4 +- apps/web/lib/coursemap/admin-operations.ts | 438 ++++++++++++++++++ .../lib/coursemap/requisite-search-actions.ts | 7 +- apps/web/playwright/access.spec.mjs | 4 + apps/web/tests/operations-format.test.ts | 38 ++ apps/web/tests/operations-sync-views.test.tsx | 222 +++++++++ .../ui/admin/catalogue/catalogue-pages.tsx | 4 +- .../catalogue/changelog/version-page.tsx | 4 +- apps/web/ui/admin/catalogue/record-header.tsx | 19 + apps/web/ui/admin/catalogue/record-page.tsx | 4 +- apps/web/ui/admin/operations/artefact-data.ts | 47 ++ .../artefact-navigation.module.css | 0 .../ui/admin/operations/artefact-viewer.tsx | 164 +++++++ .../ui/admin/operations/artefact-viewport.tsx | 31 ++ .../ui/admin/operations/discovery-detail.tsx | 105 +++++ .../ui/admin/operations/discovery-list.tsx | 89 ++++ .../ui/admin/operations/operations-format.ts | 55 +++ .../ui/admin/operations/operations-pages.tsx | 100 ++++ .../ui/admin/operations/operations-tabs.tsx | 47 ++ .../admin/operations/source-code.module.css | 48 ++ apps/web/ui/admin/operations/source-code.tsx | 46 ++ apps/web/ui/admin/operations/sync-detail.tsx | 335 ++++++++++++++ apps/web/ui/admin/operations/sync-list.tsx | 124 +++++ apps/web/ui/admin/operations/use-artefact.ts | 56 +++ apps/web/ui/shell/app-sidebar.tsx | 10 + apps/web/ui/shell/breadcrumbs.tsx | 3 + apps/web/ui/shell/notifications-menu.tsx | 2 + docs/architecture.md | 3 +- docs/catalogue-completion-plan.md | 9 +- docs/catalogue-operations.md | 23 + ...926100000_catalogue_sync_notifications.sql | 91 ++++ .../database/catalogue_sync_operations.sql | 189 ++++++++ 40 files changed, 2423 insertions(+), 34 deletions(-) create mode 100644 apps/web/app/admin/operations/catalogue/[[...section]]/page.tsx create mode 100644 apps/web/app/api/admin/catalogue-syncs/artifacts/[artifactId]/route.ts create mode 100644 apps/web/lib/coursemap/admin-operations.ts create mode 100644 apps/web/tests/operations-format.test.ts create mode 100644 apps/web/tests/operations-sync-views.test.tsx create mode 100644 apps/web/ui/admin/operations/artefact-data.ts rename apps/web/ui/admin/{catalogue => operations}/artefact-navigation.module.css (100%) create mode 100644 apps/web/ui/admin/operations/artefact-viewer.tsx create mode 100644 apps/web/ui/admin/operations/artefact-viewport.tsx create mode 100644 apps/web/ui/admin/operations/discovery-detail.tsx create mode 100644 apps/web/ui/admin/operations/discovery-list.tsx create mode 100644 apps/web/ui/admin/operations/operations-format.ts create mode 100644 apps/web/ui/admin/operations/operations-pages.tsx create mode 100644 apps/web/ui/admin/operations/operations-tabs.tsx create mode 100644 apps/web/ui/admin/operations/source-code.module.css create mode 100644 apps/web/ui/admin/operations/source-code.tsx create mode 100644 apps/web/ui/admin/operations/sync-detail.tsx create mode 100644 apps/web/ui/admin/operations/sync-list.tsx create mode 100644 apps/web/ui/admin/operations/use-artefact.ts create mode 100644 supabase/migrations/20260926100000_catalogue_sync_notifications.sql create mode 100644 supabase/tests/database/catalogue_sync_operations.sql diff --git a/apps/web/app/admin/operations/catalogue/[[...section]]/page.tsx b/apps/web/app/admin/operations/catalogue/[[...section]]/page.tsx new file mode 100644 index 00000000..fab12b05 --- /dev/null +++ b/apps/web/app/admin/operations/catalogue/[[...section]]/page.tsx @@ -0,0 +1,47 @@ +import { notFound } from "next/navigation"; +import type { SearchParams } from "@/ui/admin/catalogue/catalogue-pages"; +import { + CatalogueDiscoveryDetailPage, + CatalogueOperationsPage, + CatalogueSyncDetailPage, +} from "@/ui/admin/operations/operations-pages"; + +export const dynamic = "force-dynamic"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + +export default async function Page({ + params, + searchParams, +}: { + params: Promise<{ section?: string[] }>; + searchParams: SearchParams; +}) { + const { section = [] } = await params; + if (section.length === 0) { + return ( + + ); + } + if (section.length === 1 && section[0] === "discovery") { + return ( + + ); + } + if (section.length === 2 && section[0] === "syncs") { + if (!UUID.test(section[1]!)) notFound(); + return ; + } + if (section.length === 2 && section[0] === "discovery") { + const checkId = Number(section[1]); + if (!Number.isInteger(checkId) || checkId < 1) notFound(); + return ; + } + notFound(); +} diff --git a/apps/web/app/admin/page.tsx b/apps/web/app/admin/page.tsx index 595d17a7..69177eb9 100644 --- a/apps/web/app/admin/page.tsx +++ b/apps/web/app/admin/page.tsx @@ -2,7 +2,7 @@ import { UsersRound } from "lucide-react"; import { ImportModelCard } from "@/ui/admin/imports/import-model-card"; import { loadImportModelSetting } from "@/lib/admin/settings"; import { loadAdminUserSummary } from "@/lib/admin/users"; -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { AppShell } from "@/ui/shell"; import { StatTile } from "@/ui/common/stat-tile"; @@ -12,7 +12,7 @@ export default async function AdminOverviewPage() { const [users, importModel, canManageImports] = await Promise.all([ loadAdminUserSummary(), loadImportModelSetting(), - canManageCatalogueSources(), + canManageCatalogueOperations(), ]); return ( diff --git a/apps/web/app/api/admin/catalogue-directory/route.ts b/apps/web/app/api/admin/catalogue-directory/route.ts index d758d7c7..f14095c0 100644 --- a/apps/web/app/api/admin/catalogue-directory/route.ts +++ b/apps/web/app/api/admin/catalogue-directory/route.ts @@ -1,4 +1,4 @@ -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { refreshCatalogueDirectory } from "@/lib/catalogue-import/directory"; import { isCatalogueKind } from "@/lib/catalogue/content"; @@ -20,7 +20,7 @@ function eventResponse(data: unknown, status: number) { /** Streams directory refresh progress as server-sent events. */ export async function POST(request: Request) { - if (!(await canManageCatalogueSources())) { + if (!(await canManageCatalogueOperations())) { return eventResponse( { type: "error", message: "Import permission is required." }, 403, diff --git a/apps/web/app/api/admin/catalogue-syncs/artifacts/[artifactId]/route.ts b/apps/web/app/api/admin/catalogue-syncs/artifacts/[artifactId]/route.ts new file mode 100644 index 00000000..a4012035 --- /dev/null +++ b/apps/web/app/api/admin/catalogue-syncs/artifacts/[artifactId]/route.ts @@ -0,0 +1,49 @@ +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; +import { + type SyncArtifactLocator, + readSyncArtifact, +} from "@/lib/catalogue-sync/artifact-store"; +import { createClient } from "@/lib/supabase/server"; + +export const runtime = "nodejs"; + +/** Serves a stored sync artefact as inert text for the operations viewer. */ +export async function GET( + _request: Request, + { params }: { params: Promise<{ artifactId: string }> }, +) { + if (!(await canManageCatalogueOperations())) { + return new Response("Catalogue operations permission is required.", { + status: 403, + }); + } + const { artifactId } = await params; + const supabase = await createClient(); + const { data, error } = await supabase + .from("catalogue_sync_artifacts") + .select("media_type,content_sha256,byte_size,storage_bucket,storage_path") + .eq("id", artifactId) + .maybeSingle(); + if (error || !data) + return new Response("Artefact not found.", { status: 404 }); + try { + const body = await readSyncArtifact({ + artifact: { + bucket: data.storage_bucket as SyncArtifactLocator["bucket"], + path: data.storage_path, + mediaType: data.media_type, + contentSha256: data.content_sha256, + byteSize: data.byte_size, + }, + }); + return new Response(body, { + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "private, no-store", + "x-content-type-options": "nosniff", + }, + }); + } catch { + return new Response("The artefact could not be read.", { status: 502 }); + } +} diff --git a/apps/web/app/api/admin/catalogue-syncs/route.ts b/apps/web/app/api/admin/catalogue-syncs/route.ts index 1a46034d..5271e2aa 100644 --- a/apps/web/app/api/admin/catalogue-syncs/route.ts +++ b/apps/web/app/api/admin/catalogue-syncs/route.ts @@ -1,5 +1,5 @@ import { after } from "next/server"; -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { isCatalogueKind } from "@/lib/catalogue/content"; import { processCatalogueSyncInline } from "@/lib/catalogue-sync/sync-queue"; import { startCatalogueSync } from "@/lib/catalogue-sync/sync-service"; @@ -16,7 +16,7 @@ function json(data: unknown, status = 200) { /** Creates one record sync. Inline processing continues after the response. */ export async function POST(request: Request) { - if (!(await canManageCatalogueSources())) { + if (!(await canManageCatalogueOperations())) { return json({ error: "Catalogue sync permission is required." }, 403); } let payload: StartRequest; @@ -52,7 +52,7 @@ export async function POST(request: Request) { /** Stops an unfinished record sync. */ export async function DELETE(request: Request) { - if (!(await canManageCatalogueSources())) { + if (!(await canManageCatalogueOperations())) { return json({ error: "Catalogue sync permission is required." }, 403); } let payload: { syncId?: unknown }; diff --git a/apps/web/lib/admin/settings-actions.ts b/apps/web/lib/admin/settings-actions.ts index 1b2a4ef2..eb0e03dc 100644 --- a/apps/web/lib/admin/settings-actions.ts +++ b/apps/web/lib/admin/settings-actions.ts @@ -1,7 +1,7 @@ "use server"; import { revalidatePath } from "next/cache"; -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { createClient } from "@/lib/supabase/server"; import { IMPORT_MODEL_SETTING_KEY } from "@/lib/admin/settings"; import { fetchCatalogueModel } from "@/lib/admin/model-catalogue"; @@ -20,7 +20,7 @@ function refreshImportPages() { export async function setImportModel( model: string, ): Promise { - if (!(await canManageCatalogueSources())) + if (!(await canManageCatalogueOperations())) return { ok: false, model, @@ -61,7 +61,7 @@ export async function saveImportModel( model: string, refreshOnly = false, ): Promise { - if (!(await canManageCatalogueSources())) + if (!(await canManageCatalogueOperations())) return { ok: false, model, @@ -103,7 +103,7 @@ export async function saveImportModel( export async function removeImportModel( model: string, ): Promise { - if (!(await canManageCatalogueSources())) + if (!(await canManageCatalogueOperations())) return { ok: false, model, @@ -139,7 +139,7 @@ export async function setImportModelVisibility( model: string, visible: boolean, ): Promise { - if (!(await canManageCatalogueSources())) + if (!(await canManageCatalogueOperations())) return { ok: false, model, diff --git a/apps/web/lib/assistant/model-actions.ts b/apps/web/lib/assistant/model-actions.ts index e1af6c71..6fc181a5 100644 --- a/apps/web/lib/assistant/model-actions.ts +++ b/apps/web/lib/assistant/model-actions.ts @@ -1,6 +1,6 @@ "use server"; -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { loadImportModelSetting } from "@/lib/admin/settings"; import type { ImportModel } from "@/lib/admin/import-model"; @@ -9,7 +9,7 @@ export async function loadAssistantModels(): Promise<{ defaultModel: string; error: string | null; }> { - if (!(await canManageCatalogueSources())) { + if (!(await canManageCatalogueOperations())) { return { models: [], defaultModel: "", diff --git a/apps/web/lib/auth/viewer.ts b/apps/web/lib/auth/viewer.ts index f8d095e3..3e6db8e3 100644 --- a/apps/web/lib/auth/viewer.ts +++ b/apps/web/lib/auth/viewer.ts @@ -69,12 +69,12 @@ async function currentUserHasPermission(requiredPermission: string) { } /** Check the permission required to run programme and calendar imports. */ -export async function canManageCatalogueImports() { - return currentUserHasPermission("imports.manage"); -} - -/** Check the shared permission for catalogue sources and extraction models. */ -export async function canManageCatalogueSources() { +/** + * The permission for ANU operations: running syncs and discovery, choosing an + * extraction model, and reading the technical record of either. Separate from + * catalogue.write, which authors and publishes content. + */ +export async function canManageCatalogueOperations() { return currentUserHasPermission("imports.manage"); } diff --git a/apps/web/lib/catalogue-sync/sync-service.ts b/apps/web/lib/catalogue-sync/sync-service.ts index 7b73fe0a..d3d6ec7f 100644 --- a/apps/web/lib/catalogue-sync/sync-service.ts +++ b/apps/web/lib/catalogue-sync/sync-service.ts @@ -1,5 +1,5 @@ import "server-only"; -import { canManageCatalogueSources, getAuthViewer } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations, getAuthViewer } from "@/lib/auth/viewer"; import { loadImportModelSetting } from "@/lib/admin/settings"; import { createClient } from "@/lib/supabase/server"; import { dispatchCatalogueSync } from "./sync-queue"; @@ -24,7 +24,7 @@ export async function startCatalogueSync({ requestedBy?: string; kind: CatalogueKind; }) { - if (!(await canManageCatalogueSources())) { + if (!(await canManageCatalogueOperations())) { throw new CatalogueSyncStartError("Catalogue sync permission is required."); } const viewer = await getAuthViewer(); diff --git a/apps/web/lib/coursemap/admin-operations.ts b/apps/web/lib/coursemap/admin-operations.ts new file mode 100644 index 00000000..0c406924 --- /dev/null +++ b/apps/web/lib/coursemap/admin-operations.ts @@ -0,0 +1,438 @@ +import "server-only"; +import type { CatalogueKind } from "@/lib/catalogue/content"; +import { createClient } from "@/lib/supabase/server"; + +export const OPERATIONS_PAGE_SIZE = 25; + +export type SyncOperationRow = { + id: string; + code: string; + kind: CatalogueKind; + academicYear: number; + status: string; + trigger: "manual" | "scheduled"; + requestedAt: string; + startedAt: string | null; + completedAt: string | null; + durationMs: number | null; + attemptCount: number; + model: string | null; + costUsd: number; + errorCode: string | null; +}; + +export type SyncOperationsPage = { + rows: SyncOperationRow[]; + total: number; + page: number; + pageSize: number; + query: string; + status: string; +}; + +function durationMs(startedAt: string | null, completedAt: string | null) { + if (!startedAt || !completedAt) return null; + return Date.parse(completedAt) - Date.parse(startedAt); +} + +/** The technical list of ANU syncs, newest first, across every record. */ +export async function loadSyncOperationsPage({ + query = "", + status = "all", + page = 1, +}: { + query?: string; + status?: string; + page?: number; +}): Promise { + const supabase = await createClient(); + let request = supabase + .from("catalogue_syncs") + .select( + "id,record_id,status,trigger,requested_model,requested_at,started_at,completed_at,attempt_count,error_code,catalogue_records!inner(kind,academic_years!inner(year),catalogue_codes!inner(code))", + { count: "exact" }, + ) + .order("requested_at", { ascending: false }); + if (status !== "all") request = request.eq("status", status); + const needle = query.trim().toUpperCase(); + if (needle) { + request = request.ilike( + "catalogue_records.catalogue_codes.code", + `%${needle}%`, + ); + } + const safePage = Math.max(1, page); + const from = (safePage - 1) * OPERATIONS_PAGE_SIZE; + const { data, error, count } = await request.range( + from, + from + OPERATIONS_PAGE_SIZE - 1, + ); + if (error) throw error; + + const syncIds = (data ?? []).map((sync) => sync.id); + const { data: extractions, error: extractionError } = syncIds.length + ? await supabase + .from("catalogue_extractions") + .select("sync_id,resolved_model,requested_model,cost_usd") + .in("sync_id", syncIds) + : { data: [], error: null }; + if (extractionError) throw extractionError; + const costBySync = new Map(); + for (const extraction of extractions ?? []) { + const current = costBySync.get(extraction.sync_id) ?? { + cost: 0, + model: null, + }; + current.cost += Number(extraction.cost_usd ?? 0); + current.model = + extraction.resolved_model ?? extraction.requested_model ?? current.model; + costBySync.set(extraction.sync_id, current); + } + + return { + rows: (data ?? []).map((sync) => { + const record = sync.catalogue_records; + const usage = costBySync.get(sync.id); + return { + id: sync.id, + code: record.catalogue_codes.code, + kind: record.kind as CatalogueKind, + academicYear: record.academic_years.year, + status: sync.status, + trigger: sync.trigger as "manual" | "scheduled", + requestedAt: sync.requested_at, + startedAt: sync.started_at, + completedAt: sync.completed_at, + durationMs: durationMs(sync.started_at, sync.completed_at), + attemptCount: sync.attempt_count, + model: usage?.model ?? sync.requested_model, + costUsd: usage?.cost ?? 0, + errorCode: sync.error_code, + } satisfies SyncOperationRow; + }), + total: count ?? 0, + page: safePage, + pageSize: OPERATIONS_PAGE_SIZE, + query, + status, + }; +} + +export type SyncStage = { + id: string; + stageName: string; + attemptNumber: number; + status: string; + startedAt: string; + completedAt: string | null; + durationMs: number | null; + errorCode: string | null; + errorSummary: string | null; +}; + +export type SyncArtefact = { + id: string; + kind: string; + attemptNumber: number; + mediaType: string; + byteSize: number; +}; + +export type SyncExtraction = { + id: string; + extractionNumber: number; + requestedModel: string; + resolvedModel: string | null; + reusedFromExtractionId: string | null; + validationStatus: string; + schemaValid: boolean | null; + domainValid: boolean | null; + warningCount: number; + errorCount: number; + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + reasoningTokens: number; + costUsd: number; + costSource: string; + latencyMs: number | null; + finishReason: string | null; + errorSummary: string | null; +}; + +export type SyncDetail = { + id: string; + code: string; + kind: CatalogueKind; + academicYear: number; + recordId: number; + status: string; + trigger: "manual" | "scheduled"; + requestedModel: string; + parserVersion: string; + promptVersion: string; + schemaVersion: string; + requestedAt: string; + startedAt: string | null; + checkedAt: string | null; + completedAt: string | null; + attemptCount: number; + workerId: string | null; + leaseExpiresAt: string | null; + queueMessageId: string | null; + dispatchedAt: string | null; + errorCode: string | null; + errorMessage: string | null; + sourceVersionId: number | null; + previousSourceVersionId: number | null; + sourceDocument: { + canonicalUrl: string; + contentSha256: string; + httpStatus: number | null; + fetchedAt: string; + byteSize: number | null; + mediaType: string; + } | null; + stages: SyncStage[]; + artefacts: SyncArtefact[]; + extractions: SyncExtraction[]; + changeCount: number; +}; + +/** Everything recorded about one sync, for a developer reading a failure. */ +export async function loadSyncDetail( + syncId: string, +): Promise { + const supabase = await createClient(); + const { data: sync, error } = await supabase + .from("catalogue_syncs") + .select( + "*,catalogue_records!inner(id,kind,academic_years!inner(year),catalogue_codes!inner(code))", + ) + .eq("id", syncId) + .maybeSingle(); + if (error) throw error; + if (!sync) return null; + + const [stages, artefacts, extractions, document, changes] = await Promise.all( + [ + supabase + .from("catalogue_sync_stages") + .select("*") + .eq("sync_id", syncId) + .order("started_at"), + supabase + .from("catalogue_sync_artifacts") + .select("id,kind,attempt_number,media_type,byte_size") + .eq("sync_id", syncId) + .order("created_at"), + supabase + .from("catalogue_extractions") + .select("*") + .eq("sync_id", syncId) + .order("extraction_number"), + sync.source_document_id + ? supabase + .from("catalogue_source_documents") + .select( + "canonical_url,content_sha256,http_status,fetched_at,byte_size,media_type", + ) + .eq("id", sync.source_document_id) + .maybeSingle() + : { data: null, error: null }, + supabase + .from("catalogue_sync_changes") + .select("id", { count: "exact", head: true }) + .eq("sync_id", syncId), + ], + ); + if (stages.error) throw stages.error; + if (artefacts.error) throw artefacts.error; + if (extractions.error) throw extractions.error; + if (document.error) throw document.error; + if (changes.error) throw changes.error; + + const record = sync.catalogue_records; + return { + id: sync.id, + code: record.catalogue_codes.code, + kind: record.kind as CatalogueKind, + academicYear: record.academic_years.year, + recordId: record.id, + status: sync.status, + trigger: sync.trigger as "manual" | "scheduled", + requestedModel: sync.requested_model, + parserVersion: sync.parser_version, + promptVersion: sync.prompt_version, + schemaVersion: sync.schema_version, + requestedAt: sync.requested_at, + startedAt: sync.started_at, + checkedAt: sync.checked_at, + completedAt: sync.completed_at, + attemptCount: sync.attempt_count, + workerId: sync.worker_id, + leaseExpiresAt: sync.lease_expires_at, + queueMessageId: sync.queue_message_id, + dispatchedAt: sync.dispatched_at, + errorCode: sync.error_code, + errorMessage: sync.error_message, + sourceVersionId: sync.source_version_id, + previousSourceVersionId: sync.previous_source_version_id, + sourceDocument: document.data + ? { + canonicalUrl: document.data.canonical_url, + contentSha256: document.data.content_sha256, + httpStatus: document.data.http_status, + fetchedAt: document.data.fetched_at, + byteSize: document.data.byte_size, + mediaType: document.data.media_type, + } + : null, + stages: (stages.data ?? []).map((stage) => ({ + id: stage.id, + stageName: stage.stage_name, + attemptNumber: stage.attempt_number, + status: stage.status, + startedAt: stage.started_at, + completedAt: stage.completed_at, + durationMs: durationMs(stage.started_at, stage.completed_at), + errorCode: stage.error_code, + errorSummary: stage.error_summary, + })), + artefacts: (artefacts.data ?? []).map((artefact) => ({ + id: artefact.id, + kind: artefact.kind, + attemptNumber: artefact.attempt_number, + mediaType: artefact.media_type, + byteSize: artefact.byte_size, + })), + extractions: (extractions.data ?? []).map((extraction) => ({ + id: extraction.id, + extractionNumber: extraction.extraction_number, + requestedModel: extraction.requested_model, + resolvedModel: extraction.resolved_model, + reusedFromExtractionId: extraction.reused_from_extraction_id, + validationStatus: extraction.validation_status, + schemaValid: extraction.schema_valid, + domainValid: extraction.domain_valid, + warningCount: extraction.warning_count, + errorCount: extraction.error_count, + inputTokens: extraction.input_tokens, + cachedInputTokens: extraction.cached_input_tokens, + outputTokens: extraction.output_tokens, + reasoningTokens: extraction.reasoning_tokens, + costUsd: Number(extraction.cost_usd ?? 0), + costSource: extraction.cost_source, + latencyMs: extraction.latency_ms, + finishReason: extraction.finish_reason, + errorSummary: extraction.error_summary, + })), + changeCount: changes.count ?? 0, + }; +} + +export type DiscoveryCheckRow = { + id: number; + kind: CatalogueKind; + academicYear: number; + status: string; + isComplete: boolean; + discoveredCount: number; + startedAt: string; + completedAt: string | null; + durationMs: number | null; + errorCode: string | null; + errorMessage: string | null; +}; + +/** Recent ANU listing checks, newest first. */ +export async function loadDiscoveryChecks( + limit = OPERATIONS_PAGE_SIZE, +): Promise { + const supabase = await createClient(); + const { data, error } = await supabase + .from("catalogue_discovery_checks") + .select("*,academic_years!inner(year)") + .order("started_at", { ascending: false }) + .limit(limit); + if (error) throw error; + return (data ?? []).map((check) => ({ + id: check.id, + kind: check.kind as CatalogueKind, + academicYear: check.academic_years.year, + status: check.status, + isComplete: check.is_complete, + discoveredCount: check.discovered_count, + startedAt: check.started_at, + completedAt: check.completed_at, + durationMs: durationMs(check.started_at, check.completed_at), + errorCode: check.error_code, + errorMessage: check.error_message, + })); +} + +export type DiscoveryCheckDetail = DiscoveryCheckRow & { + retiredCount: number; + listedCount: number; + sourcePages: Array<{ + id: number; + canonicalUrl: string; + httpStatus: number | null; + fetchedAt: string; + contentSha256: string; + }>; +}; + +/** One listing check, with the pages it read and what it concluded. */ +export async function loadDiscoveryCheck( + checkId: number, +): Promise { + const supabase = await createClient(); + const { data: check, error } = await supabase + .from("catalogue_discovery_checks") + .select("*,academic_years!inner(year)") + .eq("id", checkId) + .maybeSingle(); + if (error) throw error; + if (!check) return null; + + const [pages, listings] = await Promise.all([ + supabase + .from("catalogue_discovery_check_source_pages") + .select( + "source_page_id,catalogue_source_pages!inner(id,canonical_url,http_status,fetched_at,content_sha256)", + ) + .eq("discovery_check_id", checkId), + supabase + .from("catalogue_listings") + .select("is_current", { count: "exact" }) + .eq("academic_year_id", check.academic_year_id) + .eq("kind", check.kind), + ]); + if (pages.error) throw pages.error; + if (listings.error) throw listings.error; + + const rows = listings.data ?? []; + return { + id: check.id, + kind: check.kind as CatalogueKind, + academicYear: check.academic_years.year, + status: check.status, + isComplete: check.is_complete, + discoveredCount: check.discovered_count, + startedAt: check.started_at, + completedAt: check.completed_at, + durationMs: durationMs(check.started_at, check.completed_at), + errorCode: check.error_code, + errorMessage: check.error_message, + listedCount: rows.filter((listing) => listing.is_current).length, + retiredCount: rows.filter((listing) => !listing.is_current).length, + sourcePages: (pages.data ?? []).map((page) => ({ + id: page.catalogue_source_pages.id, + canonicalUrl: page.catalogue_source_pages.canonical_url, + httpStatus: page.catalogue_source_pages.http_status, + fetchedAt: page.catalogue_source_pages.fetched_at, + contentSha256: page.catalogue_source_pages.content_sha256, + })), + }; +} diff --git a/apps/web/lib/coursemap/requisite-search-actions.ts b/apps/web/lib/coursemap/requisite-search-actions.ts index 054f7ba3..d74a9543 100644 --- a/apps/web/lib/coursemap/requisite-search-actions.ts +++ b/apps/web/lib/coursemap/requisite-search-actions.ts @@ -1,8 +1,7 @@ "use server"; import { - canManageCatalogueImports, - canManageCatalogueSources, + canManageCatalogueOperations, canWriteCourses, } from "@/lib/auth/viewer"; import { createClient } from "@/lib/supabase/server"; @@ -45,7 +44,7 @@ export async function searchRequisiteCourses( ): Promise { const term = query.trim().toUpperCase(); if (term.length < 2) return []; - if (!(await canWriteCourses()) && !(await canManageCatalogueSources())) { + if (!(await canWriteCourses()) && !(await canManageCatalogueOperations())) { return []; } @@ -89,7 +88,7 @@ export async function searchRequisiteProgrammes( ): Promise { const term = query.trim().toUpperCase(); if (term.length < 2) return []; - if (!(await canManageCatalogueImports())) return []; + if (!(await canManageCatalogueOperations())) return []; try { const supabase = await createClient(); diff --git a/apps/web/playwright/access.spec.mjs b/apps/web/playwright/access.spec.mjs index 1f961c5b..1f454cba 100644 --- a/apps/web/playwright/access.spec.mjs +++ b/apps/web/playwright/access.spec.mjs @@ -71,6 +71,9 @@ test("redirects protected routes to the canonical login page", async ({ "/admin/users", "/admin/roles", "/admin/users/70000000-0000-4000-8000-000000000001", + "/admin/operations/catalogue", + "/admin/operations/catalogue/discovery", + "/admin/operations/catalogue/syncs/11111111-1111-4111-8111-111111111111", ]) { const response = await request(api, path, { redirect: "manual" }); @@ -138,6 +141,7 @@ test("removed reference routes stay unavailable without a session", async ({ "/admin/design-system/components", "/admin/design-system-preview/tokens/foundations", "/api/design-system/review", + "/structures/bcomp", ]) { assert.equal( (await request(api, path, { redirect: "manual" })).status, diff --git a/apps/web/tests/operations-format.test.ts b/apps/web/tests/operations-format.test.ts new file mode 100644 index 00000000..e9069d46 --- /dev/null +++ b/apps/web/tests/operations-format.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; + +import { + formatBytes, + formatCost, + formatDuration, + syncStatusLabel, + syncStatusTone, +} from "../ui/admin/operations/operations-format.ts"; + +test("durations read at the scale they happened on", () => { + assert.equal(formatDuration(null), "—"); + assert.equal(formatDuration(340), "340 ms"); + assert.equal(formatDuration(1500), "1.5 s"); + assert.equal(formatDuration(95_000), "1m 35s"); +}); + +test("a fraction of a cent is still reported, because budgets are the point", () => { + assert.equal(formatCost(0), "—"); + assert.equal(formatCost(0.0031), "US$0.0031"); + assert.equal(formatCost(1.5), "US$1.50"); +}); + +test("artefact sizes read in the unit that fits", () => { + assert.equal(formatBytes(null), "—"); + assert.equal(formatBytes(900), "900 B"); + assert.equal(formatBytes(2048), "2.0 kB"); + assert.equal(formatBytes(3 * 1024 * 1024), "3.0 MB"); +}); + +test("a failed sync reads as a failure and an unknown status is not dressed up", () => { + assert.equal(syncStatusTone("failed"), "danger"); + assert.equal(syncStatusTone("review_required"), "warning"); + assert.equal(syncStatusTone("unchanged"), "success"); + assert.equal(syncStatusTone("something_new"), "neutral"); + assert.equal(syncStatusLabel("review_required"), "review required"); +}); diff --git a/apps/web/tests/operations-sync-views.test.tsx b/apps/web/tests/operations-sync-views.test.tsx new file mode 100644 index 00000000..a82f38e7 --- /dev/null +++ b/apps/web/tests/operations-sync-views.test.tsx @@ -0,0 +1,222 @@ +import { render, screen, within } from "@testing-library/react"; +import { expect, test, vi } from "vitest"; +import { TooltipProvider } from "@coursemap/ui/primitives/tooltip"; + +import type { + DiscoveryCheckRow, + SyncDetail, + SyncOperationsPage, +} from "@/lib/coursemap/admin-operations"; +import { DiscoveryList } from "@/ui/admin/operations/discovery-list"; +import { SyncDetailView } from "@/ui/admin/operations/sync-detail"; +import { SyncList } from "@/ui/admin/operations/sync-list"; + +vi.mock("next/navigation", () => ({ + usePathname: () => "/admin/operations/catalogue", + useRouter: () => ({ push: vi.fn(), replace: vi.fn(), refresh: vi.fn() }), + useSearchParams: () => new URLSearchParams(), +})); + +vi.mock("@/ui/admin/operations/artefact-viewer", () => ({ + ArtefactViewer: ({ artifacts }: { artifacts: unknown[] }) => ( +
{artifacts.length}
+ ), +})); + +function syncPage( + overrides: Partial = {}, +): SyncOperationsPage { + return { + rows: [ + { + id: "11111111-1111-4111-8111-111111111111", + code: "COMP2700", + kind: "course", + academicYear: 2027, + status: "failed", + trigger: "scheduled", + requestedAt: "2026-09-21T10:00:00.000Z", + startedAt: "2026-09-21T10:00:05.000Z", + completedAt: "2026-09-21T10:00:35.000Z", + durationMs: 30_000, + attemptCount: 3, + model: "openai/gpt-5", + costUsd: 0.0042, + errorCode: "OPENROUTER_HTTP_500", + }, + ], + total: 1, + page: 1, + pageSize: 25, + query: "", + status: "all", + ...overrides, + }; +} + +function syncDetail(overrides: Partial = {}): SyncDetail { + return { + id: "11111111-1111-4111-8111-111111111111", + code: "COMP2700", + kind: "course", + academicYear: 2027, + recordId: 12, + status: "failed", + trigger: "manual", + requestedModel: "openai/gpt-5", + parserVersion: "course-2", + promptVersion: "course-7", + schemaVersion: "course-3", + requestedAt: "2026-09-21T10:00:00.000Z", + startedAt: "2026-09-21T10:00:05.000Z", + checkedAt: null, + completedAt: "2026-09-21T10:00:35.000Z", + attemptCount: 3, + workerId: "99999999-9999-4999-8999-999999999999", + leaseExpiresAt: "2026-09-21T10:05:00.000Z", + queueMessageId: "msg-1", + dispatchedAt: "2026-09-21T10:00:01.000Z", + errorCode: "OPENROUTER_HTTP_500", + errorMessage: "OpenRouter returned 500.", + sourceVersionId: null, + previousSourceVersionId: null, + sourceDocument: { + canonicalUrl: "https://programsandcourses.anu.edu.au/course/COMP2700", + contentSha256: "a".repeat(64), + httpStatus: 200, + fetchedAt: "2026-09-21T10:00:06.000Z", + byteSize: 2048, + mediaType: "text/html", + }, + stages: [ + { + id: "stage-1", + stageName: "source_fetch", + attemptNumber: 1, + status: "completed", + startedAt: "2026-09-21T10:00:05.000Z", + completedAt: "2026-09-21T10:00:06.000Z", + durationMs: 1000, + errorCode: null, + errorSummary: null, + }, + { + id: "stage-2", + stageName: "model_extract", + attemptNumber: 3, + status: "failed", + startedAt: "2026-09-21T10:00:30.000Z", + completedAt: "2026-09-21T10:00:35.000Z", + durationMs: 5000, + errorCode: "OPENROUTER_HTTP_500", + errorSummary: "OpenRouter returned 500.", + }, + ], + artefacts: [ + { + id: "artefact-1", + kind: "raw_html", + attemptNumber: 1, + mediaType: "text/html", + byteSize: 2048, + }, + ], + extractions: [ + { + id: "extraction-1", + extractionNumber: 1, + requestedModel: "openai/gpt-5", + resolvedModel: "openai/gpt-5-2026", + reusedFromExtractionId: null, + validationStatus: "invalid", + schemaValid: false, + domainValid: null, + warningCount: 0, + errorCount: 2, + inputTokens: 1200, + cachedInputTokens: 400, + outputTokens: 300, + reasoningTokens: 0, + costUsd: 0.0042, + costSource: "provider", + latencyMs: 4200, + finishReason: "stop", + errorSummary: null, + }, + ], + changeCount: 0, + ...overrides, + }; +} + +function renderSyncList(page: SyncOperationsPage) { + // FilterBar carries hints through the shared tooltip provider. + return render( + + + , + ); +} + +test("the sync list carries the technical columns an operator needs", () => { + renderSyncList(syncPage()); + const link = screen.getByRole("link", { name: "COMP2700" }); + expect(link.getAttribute("href")).toBe( + "/admin/operations/catalogue/syncs/11111111-1111-4111-8111-111111111111", + ); + expect(screen.getByText("failed")).toBeTruthy(); + expect(screen.getByText("scheduled")).toBeTruthy(); + expect(screen.getByText("3 attempts")).toBeTruthy(); + expect(screen.getByText("30.0 s")).toBeTruthy(); + expect(screen.getByText("US$0.0042")).toBeTruthy(); +}); + +test("an empty list says what fills it rather than showing an empty table", () => { + renderSyncList(syncPage({ rows: [], total: 0 })); + expect(screen.getByText("No syncs match")).toBeTruthy(); + expect(screen.queryByRole("table")).toBeNull(); +}); + +test("the sync detail shows the failure, the lease and the attempt that failed", () => { + render(); + expect(screen.getByText("OPENROUTER_HTTP_500")).toBeTruthy(); + // The alert and the stage that failed both name it. + expect(screen.getAllByText("OpenRouter returned 500.").length).toBe(2); + expect(screen.getByText("99999999-9999-4999-8999-999999999999")).toBeTruthy(); + const stages = screen.getByText("Model extraction").closest("tr"); + expect(within(stages!).getByText("failed")).toBeTruthy(); + expect(within(stages!).getByText("3")).toBeTruthy(); + expect(screen.getByText("openai/gpt-5-2026")).toBeTruthy(); + expect(screen.getByText("2 errors")).toBeTruthy(); + expect(screen.getByTestId("artefacts").textContent).toBe("1"); +}); + +test("the sync detail links back to the record it checked", () => { + render(); + expect( + screen.getByRole("link", { name: /Open the record/ }).getAttribute("href"), + ).toBe("/admin/courses/2027/comp2700"); +}); + +test("an incomplete listing check says so, because it cannot retire anything", () => { + const checks: DiscoveryCheckRow[] = [ + { + id: 7, + kind: "course", + academicYear: 2027, + status: "completed", + isComplete: false, + discoveredCount: 120, + startedAt: "2026-09-21T10:00:00.000Z", + completedAt: "2026-09-21T10:00:20.000Z", + durationMs: 20_000, + errorCode: null, + errorMessage: null, + }, + ]; + render(); + expect(screen.getByText("Partial")).toBeTruthy(); + expect( + screen.getByRole("link", { name: "course" }).getAttribute("href"), + ).toBe("/admin/operations/catalogue/discovery/7"); +}); diff --git a/apps/web/ui/admin/catalogue/catalogue-pages.tsx b/apps/web/ui/admin/catalogue/catalogue-pages.tsx index a322c7bc..928a2c21 100644 --- a/apps/web/ui/admin/catalogue/catalogue-pages.tsx +++ b/apps/web/ui/admin/catalogue/catalogue-pages.tsx @@ -1,5 +1,5 @@ import { Suspense } from "react"; -import { canManageCatalogueSources } from "@/lib/auth/viewer"; +import { canManageCatalogueOperations } from "@/lib/auth/viewer"; import { CATALOGUE_KIND_LABELS, type CatalogueKind, @@ -28,7 +28,7 @@ export async function CatalogueDirectoryPage({ academicYear: number; searchParams: SearchParams; }) { - if (!(await canManageCatalogueSources())) return ; + if (!(await canManageCatalogueOperations())) return ; const params = await searchParams; const labels = CATALOGUE_KIND_LABELS[kind]; const page = loadCatalogueDirectoryPage({ diff --git a/apps/web/ui/admin/catalogue/changelog/version-page.tsx b/apps/web/ui/admin/catalogue/changelog/version-page.tsx index b4ce3db0..14800fef 100644 --- a/apps/web/ui/admin/catalogue/changelog/version-page.tsx +++ b/apps/web/ui/admin/catalogue/changelog/version-page.tsx @@ -4,7 +4,7 @@ import { notFound } from "next/navigation"; import { Badge } from "@coursemap/ui/components/badge"; import { Button } from "@coursemap/ui/primitives/button"; import { - canManageCatalogueSources, + canManageCatalogueOperations, canWriteCatalogue, } from "@/lib/auth/viewer"; import { diffSnapshotWrites } from "@/lib/catalogue-import/changes"; @@ -50,7 +50,7 @@ export async function CatalogueVersionPage({ compare: string | null; }) { const [canManageImports, canWrite] = await Promise.all([ - canManageCatalogueSources(), + canManageCatalogueOperations(), canWriteCatalogue(), ]); if (!canManageImports && !canWrite) return ; diff --git a/apps/web/ui/admin/catalogue/record-header.tsx b/apps/web/ui/admin/catalogue/record-header.tsx index 05a23a07..92b30637 100644 --- a/apps/web/ui/admin/catalogue/record-header.tsx +++ b/apps/web/ui/admin/catalogue/record-header.tsx @@ -81,6 +81,14 @@ export function RecordHeader({ > View on ANU