Test pipeline integrity

.github/workflows/ci.yml

@@ -147,10 +147,151 @@ jobs:
           retention-days: 90
           if-no-files-found: ignore
 
+  # GATING: protects the CI/CD guardrails from being weakened.
+  pipeline-integrity:
+    name: Pipeline Integrity Check
+    runs-on: ubuntu-latest
+    if: always()
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check required gates are still gating
+        id: gate-check
+        if: always()
+        shell: bash
+        run: |
+          python3 - <<'PY'
+          import re
+          from pathlib import Path
+
+          text = Path(".github/workflows/ci.yml").read_text()
+          failures = []
+
+          for job in ["policy-gate", "governed-file-gate"]:
+              pattern = rf"(?ms)^  {re.escape(job)}:\n(.*?)(?=^  [A-Za-z0-9_-]+:|\Z)"
+              match = re.search(pattern, text)
+
+              if not match:
+                  failures.append(f"{job} is missing")
+                  continue
+
+              section = match.group(0)
+
+              if re.search(r"(?m)^    continue-on-error:\s*true\s*$", section):
+                  failures.append(
+                      f"{job} was weakened with continue-on-error: true"
+                  )
+
+          if failures:
+              for failure in failures:
+                  print(f"FAIL: {failure}")
+              raise SystemExit(1)
+
+          print("PASS: required gates remain blocking")
+          PY
+
+      - name: Check audit trail always runs
+        id: audit-check
+        if: always()
+        shell: bash
+        run: |
+          python3 - <<'PY'
+          import re
+          from pathlib import Path
+
+          text = Path(".github/workflows/ci.yml").read_text()
+
+          pattern = r"(?ms)^  audit-trail:\n(.*?)(?=^  [A-Za-z0-9_-]+:|\Z)"
+          match = re.search(pattern, text)
+
+          if not match:
+              print("FAIL: audit-trail job is missing")
+              raise SystemExit(1)
+
+          section = match.group(0)
+
+          if not re.search(r"(?m)^    if:\s*always\(\)\s*$", section):
+              print("FAIL: audit-trail no longer uses if: always()")
+              raise SystemExit(1)
+
+          print("PASS: audit-trail still uses if: always()")
+          PY
+
+      - name: Check change classifier still exists
+        id: classifier-check
+        if: always()
+        shell: bash
+        run: |
+          python3 - <<'PY'
+          import re
+          from pathlib import Path
+
+          text = Path(".github/workflows/ci.yml").read_text()
+
+          if not re.search(r"(?m)^  change-type-check:\s*$", text):
+              print("FAIL: change-type-check job is missing")
+              raise SystemExit(1)
+
+          print("PASS: change-type-check still exists")
+          PY
+
+      - name: Create pipeline integrity report
+        if: always()
+        shell: bash
+        env:
+          GATE_RESULT: ${{ steps.gate-check.outcome }}
+          AUDIT_RESULT: ${{ steps.audit-check.outcome }}
+          CLASSIFIER_RESULT: ${{ steps.classifier-check.outcome }}
+        run: |
+          mkdir -p ci-artifacts
+
+          python3 - <<'PY'
+          import json
+          import os
+          from pathlib import Path
+
+          results = {
+              "required_gates": os.environ.get("GATE_RESULT", "unknown"),
+              "audit_trail_always": os.environ.get("AUDIT_RESULT", "unknown"),
+              "change_classifier_exists": os.environ.get(
+                  "CLASSIFIER_RESULT",
+                  "unknown"
+              )
+          }
+
+          overall_status = (
+              "success"
+              if all(result == "success" for result in results.values())
+              else "failure"
+          )
+
+          report = {
+              "job": "pipeline-integrity",
+              "display_name": "Pipeline Integrity Check",
+              "classification": "gating",
+              "overall_status": overall_status,
+              "results": results
+          }
+
+          Path("ci-artifacts/pipeline-integrity-report.json").write_text(
+              json.dumps(report, indent=2)
+          )
+          PY
+
+      - name: Upload pipeline integrity report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pipeline-integrity-report
+          path: ci-artifacts/pipeline-integrity-report.json
+          retention-days: 90
+          if-no-files-found: error
+
   audit-trail:
     name: Audit Trail
     runs-on: ubuntu-latest
-    needs: [change-type-check, policy-gate, governed-file-gate, advisory-review]
+    needs: [change-type-check, policy-gate, governed-file-gate, advisory-review, pipeline-integrity]
     if: always()
     steps:
       - uses: actions/checkout@v4
@@ -185,4 +326,4 @@ jobs:
           name: audit-trail
           path: ci-audit-trail-*.json
           retention-days: 90
-          if-no-files-found: error
+          if-no-files-found: error

.memory/reference/api-spreadsheet-library.md

@@ -0,0 +1,24 @@
+---
+classification: internal
+project: proj-csv
+doc_type: reference
+---
+
+## Spreadsheet Library Reference
+
+The CSV export service uses **csv-stream-writer** (version 2.x) as its spreadsheet and CSV generation library. This library was selected because it supports true streaming output, has no transitive dependencies, and produces RFC 4180-compliant CSV without requiring the caller to manage quoting or escaping manually.
+
+The library is initialized once per export job with a target writable stream. Column headers are declared at initialization time and cannot be changed after the first row is written. Each row is passed to the library as a plain object whose keys match the declared header names; the library handles type coercion, special-character escaping, and line termination automatically.
+
+The library does not support XLSX or ODS output. Any future requirement to generate spreadsheet formats other than CSV will require either adding a second library or replacing csv-stream-writer with a multi-format library. That decision must go through the standard format-change approval process described in the export format decision document.
+
+Usage example:
+
+```python
+writer = CsvStreamWriter(stream, columns=["id", "title", "status", "due_date"])
+for task in task_batch:
+    writer.write_row(task)
+writer.close()
+```
+
+The library is pinned to a minor version in the service's dependency manifest. Patch updates may be applied without review. Minor or major version upgrades require a changelog review and a regression run against the export integration test suite before they are merged.

.memory/reference/cost-breakdown.md

@@ -0,0 +1,15 @@
+---
+classification: confidential
+project: proj-csv
+doc_type: finance
+---
+
+## Internal Cost Figures: CSV Export Feature
+
+This document contains confidential financial projections and actual cost data for the CSV export feature. It is restricted to finance leads and senior engineering management.
+
+The infrastructure cost for the export feature in the first quarter of operation was $4,200, broken down as follows: compute for the export workers accounted for $1,800, object storage for generated export files accounted for $900, egress bandwidth for file downloads accounted for $1,100, and monitoring and alerting overhead accounted for $400. These figures are based on an average of 3,200 export jobs per month across all projects.
+
+Projected annual cost at current growth rates is $67,000, assuming a 40 percent increase in export volume driven by new enterprise customer onboarding. The unit cost per export job is expected to decrease from $1.31 to $0.94 as batch-processing optimizations ship in Q3. Cost reduction proposals under review include moving completed export files to a cheaper storage tier after 72 hours and capping retention at 30 days, which would reduce storage costs by an estimated 35 percent.
+
+Do not share these figures outside approved channels. All cost discussions in public project documents must reference only the feature's relative priority tier, not dollar amounts.

.memory/reference/decision-csv-format.md

@@ -0,0 +1,13 @@
+---
+classification: internal
+project: proj-csv
+doc_type: decision
+---
+
+## Decision: CSV as the Export File Format for the Task List
+
+After evaluating several candidate formats including JSON, XLSX, and plain CSV, the team decided to use CSV as the standard file format for exporting the task list. CSV was chosen because it is universally supported by spreadsheet applications, requires no special libraries to open, and produces compact output that is easy to diff in version control. The format aligns with what our primary users—project managers and team leads—already use in their day-to-day tooling.
+
+Alternative formats were considered and rejected for the following reasons. JSON was ruled out because non-technical stakeholders cannot open it without additional tooling. XLSX was ruled out due to binary format complexity, licensing concerns around third-party spreadsheet libraries, and the additional dependency weight it would add to the export service. Plain text was too unstructured to be useful for downstream import workflows.
+
+The decision is considered stable. Any future proposal to change the export format must include a migration plan for existing integrations and must be approved by the product lead before implementation begins.

.memory/reference/error-codes.md

@@ -0,0 +1,21 @@
+---
+classification: internal
+project: proj-csv
+doc_type: reference
+---
+
+## Export Error Code Reference
+
+This document lists error codes produced by the CSV export service and describes their meaning and recommended remediation steps.
+
+**E_EXPORT_400** — Invalid export request. The request body failed schema validation. Check that all required fields are present and that field values match the expected types. No export job was created.
+
+**E_EXPORT_403** — Permission denied. The requesting user does not have export rights for the specified project. Contact your project administrator to have the export permission granted to your role.
+
+**E_EXPORT_404** — Project not found. The project identifier supplied in the export request does not match any known project. Verify the project ID and retry.
+
+**E_EXPORT_417** — Expectation failed during export generation. This error indicates that the export service received a request it accepted but could not fulfil because an internal precondition was not met at generation time. Common causes include a task filter that returns zero rows, a missing template configuration, or a column mapping that references a field that no longer exists in the task schema. Inspect the job detail record for the specific precondition message and correct the export configuration before retrying.
+
+**E_EXPORT_500** — Internal server error. An unexpected failure occurred inside the export service. The error has been logged automatically. If the error persists after retrying, open a support ticket and include the job ID from the error response.
+
+**E_EXPORT_503** — Export service temporarily unavailable. The service is under maintenance or experiencing high load. Retry after the interval specified in the Retry-After response header.

.memory/reference/feature-csv-export.md

@@ -0,0 +1,11 @@
+---
+classification: internal
+project: proj-csv
+doc_type: feature
+---
+
+## Feature: CSV Export
+
+The CSV export feature builds its output file using a streaming writer that processes tasks row by row without loading the entire dataset into memory. When a user requests an export, the export service opens a writable stream, writes the header row containing column names, then iterates over the filtered task set in batches of 500 records. Each task is serialized to a CSV row and flushed to the stream immediately. Once all records are written the stream is closed and the completed file is handed off to the download handler. This streaming approach keeps memory consumption flat regardless of how many tasks are exported.
+
+If CSV export generation fails at any stage, the service applies an exponential backoff retry policy. The first retry occurs after two seconds, the second after four seconds, and the third after eight seconds. After three failed attempts the job is marked as permanently failed and the user receives an error notification. Transient network errors and temporary storage unavailability are retried automatically. Validation errors and permission errors are not retried because they indicate a problem that will not resolve itself without user intervention.

.memory/reference/feature-csv-import.md

@@ -0,0 +1,13 @@
+---
+classification: internal
+project: proj-csv
+doc_type: feature
+---
+
+## Feature: CSV Import
+
+The CSV import feature allows users to bulk-load tasks into the system from a CSV file. When a user uploads a file, the import service reads it line by line, validates each row against the task schema, and inserts valid rows into the database. Rows that fail validation are collected into an error report that the user can download after the import completes.
+
+The import service enforces a maximum file size of 10 MB and a maximum row count of 5,000 tasks per import operation. Files that exceed either limit are rejected immediately with an informative error message before any rows are processed. Duplicate detection is based on the external task ID field. If a row shares an external ID with an existing task, the import service updates the existing record rather than creating a new one.
+
+Column mapping is configurable. Users may upload a column-map JSON file alongside the CSV to specify which CSV column corresponds to which task field. If no column map is provided, the import service expects the CSV header row to use the canonical field names defined in the task schema documentation.

.memory/reference/lesson-human-approval-gate.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: internal
+doc_type: lesson
+---
+
+# Lesson: Keep Human Approval Before Final Workflow Completion
+
+## What happened
+The orchestrated workflow included separate planning, implementation, review, and testing stages. Even after automated review and testing passed, the workflow still required a Human Approval step before the Project Manager treated the work as complete.
+
+## What we learned
+Automated agents can verify many technical conditions, but a successful automated result should not automatically authorize every final action. A human approval gate provides a clear checkpoint before the workflow moves to completion.
+
+## How a future developer should apply this
+For workflows that produce meaningful project changes, place human approval after automated review and testing but before final completion or release. Give the approval step the evidence produced by earlier roles so the human can make an informed decision.

.memory/reference/lesson-memory-scope-check.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: internal
+doc_type: lesson
+---
+
+# Lesson: Verify Memory Scope Before Reusing Stored Knowledge
+
+## What happened
+While building persistent memory, project decisions and reusable knowledge were stored across runs. This created a risk that an agent could accidentally reuse information that belonged to a different project or workflow.
+
+## What we learned
+Persistent memory is only trustworthy when scope is checked before use. A stored entry may be valid in one project but inappropriate in another, so project boundaries must be enforced before reading or applying remembered information.
+
+## How a future developer should apply this
+Require every storage or retrieval request to include the correct project identifier. Before using retrieved knowledge, confirm that it belongs to the active project and that the role is allowed to access its classification level.

.memory/reference/lesson-readonly-reviewer.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: internal
+doc_type: lesson
+---
+
+# Lesson: Keep the Reviewer Read-Only
+
+## What happened
+During the orchestrated workflow, the Reviewer was responsible for inspecting implementation work and returning PASS or NEEDS_CHANGES. Allowing the Reviewer to modify files would blur the boundary between implementation and independent review.
+
+## What we learned
+A Reviewer should remain read-only so that review findings stay independent from the implementation being evaluated. The Reviewer can inspect files and search the codebase, but should not directly fix the code it is reviewing.
+
+## How a future developer should apply this
+When configuring a Reviewer agent, grant only the read and search capabilities required for evaluation. If the Reviewer finds a problem, return the finding to the Implementer instead of giving the Reviewer permission to change the implementation.

.memory/reference/lesson-retrieval-calibration.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: internal
+doc_type: lesson
+---
+
+# Lesson: Tune Retrieval Without Weakening the Answer Key
+
+## What happened
+During retrieval validation, the paragraph configuration passed seven of eight ground-truth queries. The spreadsheet-library query found the correct document through keyword fallback, but it did not meet the required vector similarity threshold. A semantic chunking experiment reduced the overall result to six of eight by introducing another failure.
+
+## What we learned
+Retrieval tuning should be evaluated against a fixed ground-truth set. A configuration change that improves one query but reduces overall retrieval quality is a regression. Confidence thresholds or expected answers should not be weakened simply to make a failing test pass.
+
+## How a future developer should apply this
+Run the complete ground-truth set after each retrieval change. Compare pass rates and individual failures, keep the stronger stable configuration, and document any understood gap instead of changing the expected result to match current server behavior.

.memory/reference/lesson-role-tool-scoping.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: internal
+doc_type: lesson
+---
+
+# Lesson: Scope Tools by Role Responsibility
+
+## What happened
+While designing the orchestrated workflow, each subagent needed different capabilities. Giving every role the same tools would have allowed reviewers, planners, or project managers to perform actions outside their responsibilities.
+
+## What we learned
+Tool access should be based on the role's actual responsibility rather than convenience. Read-only roles should not receive write or execution capabilities, and specialized tools should only be granted to the role that needs them.
+
+## How a future developer should apply this
+When adding a new agent or workflow step, start with the smallest possible tool grant. Add a capability only when the role cannot complete its responsibility without it, and document why that grant is necessary.

.memory/reference/lesson-sensitive-configuration.md

@@ -0,0 +1,16 @@
+---
+project: proj-lessons
+classification: confidential
+doc_type: lesson
+---
+
+# Lesson: Keep Sensitive Configuration Details Out of Broad Agent Access
+
+## What happened
+During environment setup, some runtime configuration depended on environment variables and authentication-related values. Exposing sensitive configuration details broadly to every role would create unnecessary risk because most roles do not need those details to complete their work.
+
+## What we learned
+Sensitive operational details should be classified above normal internal project knowledge and should only be available to roles that have a legitimate need to access them. Agent instructions alone are not sufficient protection; the classification ceiling must enforce the restriction.
+
+## How a future developer should apply this
+Store only sanitized operational guidance in broadly accessible project knowledge. Keep sensitive configuration details classified as confidential or higher, and verify with retrieval tests that roles operating at an internal ceiling cannot retrieve that material.