feat: show what the model flagged on the Changes tab

apps/web/lib/catalogue-sync/persist-source-version.ts

@@ -418,7 +418,10 @@ async function insertRequirements(
   }
 }
 
-/** Writes every content, requirement and evidence row for a new snapshot. */
+/**
+ * Writes every content, requirement, evidence and review-flag row for a new
+ * snapshot.
+ */
 export async function insertVersionContent(
   tx: Tx,
   {
@@ -469,6 +472,16 @@ export async function insertVersionContent(
       )
     `;
   }
+  for (const [index, flag] of write.flags.entries()) {
+    await tx`
+      insert into public.catalogue_version_flags (
+        version_id, position, field_path, severity, code, message
+      ) values (
+        ${snapshotId}, ${index + 1}, ${flag.fieldPath}, ${flag.severity},
+        ${flag.code}, ${flag.message}
+      )
+    `;
+  }
 }
 
 /** Persists one semantic ANU observation without changing local content. */

apps/web/lib/catalogue/review-notes.ts

@@ -0,0 +1,143 @@
+/** One note the model left on a source version. */
+export type VersionFlag = {
+  fieldPath: string | null;
+  severity: "warning" | "error";
+  code: string;
+  message: string;
+};
+
+/** The model's evidence for one field, and how directly the page states it. */
+export type VersionEvidence = {
+  fieldPath: string;
+  confidence: number | null;
+  excerpt: string | null;
+};
+
+export type ReviewNote = {
+  fieldPath: string | null;
+  label: string;
+  message: string;
+};
+
+export type UncertainField = {
+  fieldPath: string;
+  label: string;
+  confidence: number;
+  excerpt: string | null;
+};
+
+/** Below this, a field is worth reading against the ANU page. */
+export const UNCERTAIN_CONFIDENCE = 0.8;
+
+const FIELD_NAMES: Record<string, string> = {
+  modelExtraction: "The whole response",
+  prerequisiteRule: "Prerequisite rule",
+  corequisiteRule: "Corequisite rule",
+  prerequisiteText: "Prerequisite wording",
+  corequisiteText: "Corequisite wording",
+  incompatibilityText: "Incompatibility wording",
+  incompatibilityCourseCodes: "Incompatible courses",
+  softIncompatibilityCourseCodes: "Advisory incompatibilities",
+  unmodelledText: "Unmodelled requirement wording",
+  unitValue: "Units",
+  eftsl: "EFTSL",
+  totalUnits: "Units",
+  durationYears: "Length",
+  selectionRank: "Selection rank",
+  atar: "ATAR",
+  learningOutcomes: "Learning outcomes",
+  assessmentItems: "Assessment",
+  offerings: "Offerings",
+  summaryFields: "Key facts",
+  relationships: "Related structures",
+  requirements: "Requirements",
+  rule: "Requirement tree",
+  sourceUpdatedAt: "ANU update date",
+  areasOfInterest: "Areas of interest",
+  relatedCourses: "Related courses",
+  contactText: "Contact",
+  convenerText: "Convener",
+};
+
+function humanise(segment: string) {
+  const words = segment.replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase();
+  return words.charAt(0).toUpperCase() + words.slice(1);
+}
+
+/**
+ * A field path from the model's extraction, such as `requisites.prerequisiteRule`,
+ * `fees[2]` or `requirements.rule.children.3`, in words an administrator reads.
+ * The section a path belongs to leads; an array index becomes an item number
+ * and a requirement branch keeps its position.
+ */
+export function modelFieldLabel(fieldPath: string | null) {
+  if (!fieldPath) return "The record";
+  const segments = fieldPath.split(/[.[\]]/).filter(Boolean);
+  const names: string[] = [];
+  const branch: number[] = [];
+  let inRequirementTree = false;
+  for (const segment of segments) {
+    if (/^\d+$/.test(segment)) {
+      if (inRequirementTree) branch.push(Number(segment) + 1);
+      else names.push(`item ${Number(segment) + 1}`);
+      continue;
+    }
+    if (segment === "rule" && names.at(-1) === FIELD_NAMES.requirements) {
+      inRequirementTree = true;
+      continue;
+    }
+    if (segment === "children" || segment === "requisites") continue;
+    names.push(FIELD_NAMES[segment] ?? humanise(segment));
+  }
+  // Sentence case: only the first name keeps its capital, unless a later one
+  // is an initialism such as ATAR.
+  const label = names
+    .map((name, index) =>
+      index > 0 && /^[A-Z][a-z]/.test(name)
+        ? name.charAt(0).toLowerCase() + name.slice(1)
+        : name,
+    )
+    .join(", ");
+  return branch.length ? `${label}, branch ${branch.join(".")}` : label;
+}
+
+/**
+ * What the Changes tab asks an administrator to check: every error, every
+ * warning, and each field whose weakest evidence falls below
+ * `UNCERTAIN_CONFIDENCE`, least certain first.
+ */
+export function summariseReviewNotes({
+  flags,
+  evidence,
+}: {
+  flags: readonly VersionFlag[];
+  evidence: readonly VersionEvidence[];
+}) {
+  const note = (flag: VersionFlag): ReviewNote => ({
+    fieldPath: flag.fieldPath,
+    label: modelFieldLabel(flag.fieldPath),
+    message: flag.message,
+  });
+  const weakest = new Map<string, UncertainField>();
+  for (const item of evidence) {
+    if (item.confidence === null || item.confidence >= UNCERTAIN_CONFIDENCE) {
+      continue;
+    }
+    const current = weakest.get(item.fieldPath);
+    if (!current || item.confidence < current.confidence) {
+      weakest.set(item.fieldPath, {
+        fieldPath: item.fieldPath,
+        label: modelFieldLabel(item.fieldPath),
+        confidence: item.confidence,
+        excerpt: item.excerpt,
+      });
+    }
+  }
+  return {
+    errors: flags.filter(({ severity }) => severity === "error").map(note),
+    warnings: flags.filter(({ severity }) => severity === "warning").map(note),
+    uncertain: [...weakest.values()].sort(
+      (left, right) => left.confidence - right.confidence,
+    ),
+  };
+}

apps/web/lib/coursemap/admin-catalogue-record.ts

@@ -1,4 +1,8 @@
 import "server-only";
+import type {
+  VersionEvidence,
+  VersionFlag,
+} from "@/lib/catalogue/review-notes";
 import { readVersionContent } from "@/lib/catalogue-import/version-content";
 import { withSyncDatabaseClient } from "@/lib/catalogue-sync/sync-store";
 import type { CatalogueContent } from "@/lib/catalogue/content";
@@ -211,6 +215,46 @@ export async function loadVersionWrite(
   return withSyncDatabaseClient((sql) => readVersionContent(sql, versionId));
 }
 
+/**
+ * The flags and model evidence stored with a source version, for the Changes
+ * tab. Empty for a version an administrator published, which has neither.
+ */
+export async function loadVersionReviewNotes(versionId: number): Promise<{
+  flags: VersionFlag[];
+  evidence: VersionEvidence[];
+}> {
+  return withSyncDatabaseClient(async (sql) => {
+    const [flags, evidence] = await Promise.all([
+      sql`
+        select field_path, severity, code, message
+        from public.catalogue_version_flags
+        where version_id = ${versionId}
+        order by position
+      `,
+      sql`
+        select field_path, confidence, source_excerpt
+        from public.catalogue_version_provenance
+        where version_id = ${versionId} and method = 'model'
+        order by id
+      `,
+    ]);
+    return {
+      flags: flags.map((row) => ({
+        fieldPath: row.field_path === null ? null : String(row.field_path),
+        severity: row.severity === "error" ? "error" : "warning",
+        code: String(row.code),
+        message: String(row.message),
+      })),
+      evidence: evidence.map((row) => ({
+        fieldPath: String(row.field_path),
+        confidence: row.confidence === null ? null : Number(row.confidence),
+        excerpt:
+          row.source_excerpt === null ? null : String(row.source_excerpt),
+      })),
+    };
+  });
+}
+
 /** The student-facing course details for a version, or null for structures. */
 export async function loadVersionCoursePreview(versionId: number) {
   const supabase = await createClient();

apps/web/tests/catalogue-changes-panel.test.tsx

@@ -5,6 +5,7 @@ import type {
   SourceReview,
   SourceReviewChange,
 } from "@/lib/catalogue/source-review-store";
+import { summariseReviewNotes } from "@/lib/catalogue/review-notes";
 import { CatalogueChangesPanel } from "@/ui/admin/catalogue/changes/changes-panel";
 
 const actions = vi.hoisted(() => ({ resolve: vi.fn() }));
@@ -157,3 +158,53 @@ test("kept values stay available without nagging", () => {
   ).toBeTruthy();
   expect(screen.queryByRole("button", { name: "Keep current" })).toBeNull();
 });
+
+test("what the model flagged leads the tab, least certain field first", () => {
+  renderPanel({
+    review: review({ incoming: [change()] }),
+    notes: summariseReviewNotes({
+      flags: [
+        {
+          fieldPath: "requisites.prerequisiteRule",
+          severity: "error",
+          code: "INVALID",
+          message: "The rule named a course code ANU does not use.",
+        },
+        {
+          fieldPath: "requirements.rule.children.3",
+          severity: "warning",
+          code: "AMBIGUOUS",
+          message: "Kept as the page's wording.",
+        },
+      ],
+      evidence: [
+        { fieldPath: "fees", confidence: 0.9, excerpt: "$5520" },
+        { fieldPath: "offerings", confidence: 0.55, excerpt: "First Semester" },
+        { fieldPath: "college", confidence: 0.7, excerpt: "ANU College" },
+      ],
+    }),
+  });
+  expect(
+    screen.getByRole("heading", { name: "What to check" }),
+  ).toBeInTheDocument();
+  expect(
+    screen.getByText("1 part could not be read and was left empty"),
+  ).toBeInTheDocument();
+  expect(screen.getByText("Prerequisite rule:")).toBeInTheDocument();
+  expect(screen.getByText("Requirements, branch 4:")).toBeInTheDocument();
+  const uncertain = screen
+    .getAllByText(/% sure$/u)
+    .map((node) => node.textContent);
+  // Fees are sure enough not to be listed.
+  expect(uncertain).toEqual(["55% sure", "70% sure"]);
+});
+
+test("nothing flagged means no notes section", () => {
+  renderPanel({
+    review: review({ incoming: [change()] }),
+    notes: summariseReviewNotes({ flags: [], evidence: [] }),
+  });
+  expect(
+    screen.queryByRole("heading", { name: "What to check" }),
+  ).not.toBeInTheDocument();
+});

apps/web/tests/review-notes.test.ts

@@ -0,0 +1,42 @@
+import { expect, test } from "vitest";
+
+import {
+  modelFieldLabel,
+  summariseReviewNotes,
+} from "@/lib/catalogue/review-notes";
+
+test("model field paths read as the fields an administrator knows", () => {
+  expect(modelFieldLabel("requisites.prerequisiteRule")).toBe(
+    "Prerequisite rule",
+  );
+  expect(modelFieldLabel("fees[2]")).toBe("Fees, item 3");
+  expect(modelFieldLabel("requirements.rule.children.3.children.0")).toBe(
+    "Requirements, branch 4.1",
+  );
+  expect(modelFieldLabel("offerings[1].startsOn")).toBe(
+    "Offerings, item 2, starts on",
+  );
+  expect(modelFieldLabel("atar")).toBe("ATAR");
+  expect(modelFieldLabel("modelExtraction")).toBe("The whole response");
+  expect(modelFieldLabel(null)).toBe("The record");
+});
+
+test("a field is listed once, at its weakest evidence", () => {
+  const { uncertain } = summariseReviewNotes({
+    flags: [],
+    evidence: [
+      { fieldPath: "fees", confidence: 0.6, excerpt: "first" },
+      { fieldPath: "fees", confidence: 0.4, excerpt: "weakest" },
+      { fieldPath: "title", confidence: null, excerpt: null },
+      { fieldPath: "college", confidence: 0.8, excerpt: "at the threshold" },
+    ],
+  });
+  expect(uncertain).toEqual([
+    {
+      fieldPath: "fees",
+      label: "Fees",
+      confidence: 0.4,
+      excerpt: "weakest",
+    },
+  ]);
+});

apps/web/types/database.ts

@@ -2039,6 +2039,51 @@ export type Database = {
           },
         ]
       }
+      catalogue_version_flags: {
+        Row: {
+          code: string
+          field_path: string | null
+          id: number
+          message: string
+          position: number
+          severity: string
+          version_id: number
+        }
+        Insert: {
+          code: string
+          field_path?: string | null
+          id?: never
+          message: string
+          position: number
+          severity: string
+          version_id: number
+        }
+        Update: {
+          code?: string
+          field_path?: string | null
+          id?: never
+          message?: string
+          position?: number
+          severity?: string
+          version_id?: number
+        }
+        Relationships: [
+          {
+            foreignKeyName: "catalogue_version_flags_version_id_fkey"
+            columns: ["version_id"]
+            isOneToOne: false
+            referencedRelation: "catalogue_versions"
+            referencedColumns: ["id"]
+          },
+          {
+            foreignKeyName: "catalogue_version_flags_version_id_fkey"
+            columns: ["version_id"]
+            isOneToOne: false
+            referencedRelation: "published_course_summaries"
+            referencedColumns: ["version_id"]
+          },
+        ]
+      }
       catalogue_version_provenance: {
         Row: {
           academic_year_id: number
@@ -4272,4 +4317,3 @@ export const Constants = {
     Enums: {},
   },
 } as const
-