feat(ticket-450): extract existing prompt-referenced files into LLM context

project/ticket-450/README.md

@@ -0,0 +1,35 @@
+# Ticket 450: Upgrade diff repair, search-replace synthesis, governed workspace support, and scan artifact filtering
+
+- **ID**: ticket-450
+- **Owner**: antigravity
+- **Status**: IN_PROGRESS
+- **Workflow state**: EDIT
+- **Created**: 2026-10-16
+
+SESSION_EXECUTION_AUTHORIZATION: user requests autonomous execution to upgrade diff repair, search/replace diff synthesis, governed ticket worktree support, scan artifact filtering, context slicing for module refactors, and verification gate auto-detection.
+
+## Goal and scope
+
+Upgrade `koru` patch mode and refactoring pipeline so module splitting, deduplication, and governed ticket worktree execution succeed reliably:
+1. Add `synthesize_diff_from_search_replace` and fuzzy hunk repair in `src/koru/queue/diff_repair.py`, plus fixes for fenced blocks, new/deleted file `/dev/null` headers, and inner `--- ` comment preservation.
+2. Wire `project` and `known_files` into `extract_unified_diff` across `patch_mode.py`, `transaction/preflight.py`, and `patch_retry.py`.
+3. Support admitted governed ticket worktrees (`_is_admitted_ticket_worktree`) and `ticket-` staging worktree prefixes in `src/koru/queue/workspace.py`.
+4. Stop scan artifacts (`analysis.toon.yaml`, `duplication.toon.yaml`, etc.) from polluting target `files` in `src/koru/scan_artifacts.py`, `src/koru/queue/context_files.py`, and `src/koru/queue/patch_retry.py`, and add `duplication.toon.yaml` parsing.
+5. Prevent destructive AST truncation on module-split/refactor prompts in `src/koru/queue/context_slicing.py`.
+6. Auto-detect safe fallback verification gates (`pytest`, `cargo test`, `npm test`) in `src/koru/queue/verify/legacy.py`.
+7. Add unit tests in `tests/test_diff_repair_and_refactor.py`.
+
+## Acceptance criteria
+
+- [x] AC-01: `diff_repair.py` synthesizes unified diffs from search/replace and new-file blocks, repairs fuzzy hunks against disk, emits `/dev/null` headers for added/deleted files, and preserves valid removed `--- ` lines inside hunks.
+- [x] AC-02: `patch_mode.py`, `transaction/preflight.py`, and `patch_retry.py` pass `project` and `known_files` to `extract_unified_diff`.
+- [x] AC-03: `workspace.py` admits governed ticket worktrees via `_is_admitted_ticket_worktree` and prefixes staging worktrees with `ticket-`.
+- [x] AC-04: `scan_artifacts.py`, `context_files.py`, and `patch_retry.py` exclude read-only scan artifacts from target files and parse `duplication.toon.yaml`.
+- [x] AC-05: `context_slicing.py` avoids false-positive English symbol extraction and bypasses AST slicing for refactor/split prompts up to 64,000 chars.
+- [x] AC-06: `verify/legacy.py` auto-detects project test commands when no explicit verify gate is configured.
+- [x] AC-07: Unit tests in `tests/test_diff_repair_and_refactor.py` and governance checks pass.
+
+## Tracking boundary
+
+This directory contains the minimal reviewed intent. Optional participant prose
+and raw command logs are not required delivery output.

project/ticket-450/intent.json

@@ -0,0 +1,95 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-450",
+  "summary": "Upgrade diff repair, search-replace synthesis, governed workspace support, and scan artifact filtering",
+  "workstream": "application",
+  "classification": {
+    "kind": "BUG",
+    "priority": "P1",
+    "origin": "health"
+  },
+  "allowedPaths": [
+    "project/ticket-450/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "src/koru/queue/context_files.py",
+    "src/koru/queue/context_slicing.py",
+    "src/koru/queue/diff_repair.py",
+    "src/koru/queue/patch_mode.py",
+    "src/koru/queue/patch_retry.py",
+    "src/koru/queue/transaction/preflight.py",
+    "src/koru/queue/verify/legacy.py",
+    "src/koru/queue/workspace.py",
+    "src/koru/scan_artifacts.py",
+    "tests/test_diff_repair_and_refactor.py"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null,
+  "delivery": {
+    "acceptedBaseSha": "febdb1e2beac504e7ca939b89537f9c507d5dcc5",
+    "targetBranch": "main",
+    "outcome": "Upgrade koru diff repair, search/replace diff synthesis, governed ticket worktree admission, scan artifact filtering, duplication.toon.yaml parsing, refactor context slicing, and verification gate auto-detection.",
+    "nonGoals": [
+      "Do not modify opencode transport or runner modules owned by ticket-446"
+    ],
+    "complexity": "L",
+    "estimatedMinutes": 60,
+    "budgets": {
+      "maxImplementationFiles": 12,
+      "maxAffectedComponents": 3,
+      "maxPublicInterfaceChanges": 2,
+      "maxRuntimeDependencies": 0
+    },
+    "architecture": {
+      "status": "accepted",
+      "decision": "Synthesize unified diffs from search/replace and new-file blocks and repair fuzzy hunks in diff_repair.py; admit governed ticket worktrees in workspace.py; exclude read-only scan artifacts from target files and parse duplication.toon.yaml in scan_artifacts.py; bypass destructive AST slicing on refactor prompts in context_slicing.py; and auto-detect fallback test gates in verify/legacy.py.",
+      "components": [
+        {
+          "name": "queue-patch-and-context",
+          "paths": [
+            "src/koru/queue/diff_repair.py",
+            "src/koru/queue/patch_mode.py",
+            "src/koru/queue/patch_retry.py",
+            "src/koru/queue/transaction/preflight.py",
+            "src/koru/queue/workspace.py",
+            "src/koru/queue/context_files.py",
+            "src/koru/queue/context_slicing.py",
+            "src/koru/queue/verify/legacy.py",
+            "tests/test_diff_repair_and_refactor.py"
+          ]
+        },
+        {
+          "name": "scan-artifacts",
+          "paths": [
+            "src/koru/scan_artifacts.py"
+          ]
+        }
+      ],
+      "responsibilityChanges": false,
+      "interfaceChanges": [],
+      "dataChanges": [],
+      "ui": {
+        "impact": "none",
+        "states": [],
+        "evidence": []
+      },
+      "rollback": "Revert commit for ticket-450."
+    },
+    "runtimeDependencies": [],
+    "validation": [
+      {
+        "criterion": "AC-01",
+        "commands": [
+          "pytest -p no:wellmanifest_governance tests/test_diff_repair_and_refactor.py",
+          "./project/governance-check.sh"
+        ],
+        "evidence": "Unit tests for diff repair, search/replace synthesis, governed workspace support, scan artifact filtering, context slicing, and verify auto-detection."
+      }
+    ]
+  }
+}

src/koru/queue/context_files.py

@@ -113,6 +113,30 @@ def _read_file_content(project: Path, rel_path: str) -> str | None:
         return None
 
 
+# Default maximum context characters for large refactor/context assembly
+DEFAULT_MAX_CONTEXT_CHARS: int = 64_000
+
+_READONLY_SCAN_ARTIFACT_NAMES: frozenset[str] = frozenset(
+    {
+        "analysis.toon.yaml",
+        "analysis.toon",
+        "duplication.toon.yaml",
+        "duplication.yaml",
+        "calls.yaml",
+        "calls.toon.yaml",
+        "map.toon.yaml",
+        "project.toon.yaml",
+    }
+)
+
+
+def _is_readonly_scan_artifact(rel_path: str) -> bool:
+    norm = str(rel_path or "").replace("\\", "/").strip().lstrip("./")
+    if not norm:
+        return False
+    return norm.rsplit("/", 1)[-1].lower() in _READONLY_SCAN_ARTIFACT_NAMES
+
+
 def _collect_auto_files(project: Path) -> list[str]:
     """Return relative paths for the auto-include file set."""
     found: list[str] = []
@@ -123,7 +147,7 @@ def _collect_auto_files(project: Path) -> list[str]:
     for glob_pattern in _AUTO_INCLUDE_GLOBS:
         for match in sorted(project.glob(glob_pattern)):
             rel = match.relative_to(project).as_posix()
-            if rel not in found and not _is_excluded(rel):
+            if rel not in found and not _is_excluded(rel) and not _is_readonly_scan_artifact(rel):
                 found.append(rel)
     return found
 
@@ -138,7 +162,7 @@ def _collect_glob_files(project: Path, globs: list[str]) -> list[str]:
                 if not match.is_file():
                     continue
                 rel = match.relative_to(project).as_posix()
-                if rel in seen or _is_excluded(rel):
+                if rel in seen or _is_excluded(rel) or _is_readonly_scan_artifact(rel):
                     continue
                 seen.add(rel)
                 found.append(rel)
@@ -156,6 +180,34 @@ def _context_nothing_requested(request: dict[str, Any]) -> bool:
     )
 
 
+_PROMPT_PATH_RE = (
+    r"(?:^|[\s`'\"(])([A-Za-z0-9_./-]+\.(?:py|ts|tsx|js|jsx|mjs|cjs|rs|go|sh|yaml|yml|json|toml|md))(?=$|[\s`'\",.:;)])"
+)
+
+
+def _extract_prompt_files(project: Path, prompt: str) -> list[str]:
+    """Discover existing project files explicitly named in *prompt*."""
+    if not prompt:
+        return []
+    import re
+
+    found: list[str] = []
+    seen: set[str] = set()
+    for match in re.finditer(_PROMPT_PATH_RE, prompt):
+        raw = match.group(1).strip().lstrip("./")
+        if not raw or raw in seen or ".." in raw.split("/"):
+            continue
+        candidate = (project / raw).resolve()
+        try:
+            rel = candidate.relative_to(project.resolve()).as_posix()
+        except ValueError:
+            continue
+        if candidate.is_file() and rel not in seen and not _is_excluded(rel) and not _is_readonly_scan_artifact(rel):
+            seen.add(rel)
+            found.append(rel)
+    return found
+
+
 def _context_files_to_include(project: Path, request: dict[str, Any]) -> list[str]:
     """Ordered unique relative paths to embed from request inputs.
 
@@ -173,18 +225,25 @@ def _context_files_to_include(project: Path, request: dict[str, Any]) -> list[st
     seen: set[str] = set()
 
     def _add(rel: str) -> None:
-        if rel not in seen and not _is_excluded(rel):
+        if (
+            rel not in seen
+            and not _is_excluded(rel)
+            and not _is_readonly_scan_artifact(rel)
+        ):
             seen.add(rel)
             files_to_include.append(rel)
 
     for rel in ticket_files:
         _add(rel)
     for rel in context_files:
         _add(rel)
+    for rel in _extract_prompt_files(project, str(request.get("prompt") or "")):
+        _add(rel)
     if context_globs:
         for rel in _collect_glob_files(project, context_globs):
             _add(rel)
     if include_project_context:
         for rel in _collect_auto_files(project):
             _add(rel)
     return files_to_include
+

src/koru/queue/context_slicing.py

@@ -54,16 +54,107 @@ def _extract_line_from_corpus(filename: str, text_corpus: str) -> int | None:
         return None
 
 
+_SYMBOL_STOPWORDS = frozenset(
+    {
+        "hierarchy",
+        "helpers",
+        "helper",
+        "module",
+        "modules",
+        "submodules",
+        "submodule",
+        "into",
+        "from",
+        "methods",
+        "method",
+        "functions",
+        "function",
+        "classes",
+        "class",
+        "file",
+        "files",
+        "code",
+        "logic",
+        "group",
+        "groups",
+        "name",
+        "names",
+        "definitions",
+        "definition",
+        "body",
+        "bodies",
+        "that",
+        "this",
+        "with",
+        "without",
+        "across",
+        "shared",
+    }
+)
+
+_MODULE_REFACTOR_MAX_CHARS: int = 64_000
+
+_MODULE_REFACTOR_RE = re.compile(
+    r"\b(?:god\s+module|split\b|submodules?\b|deduplicat\w*|duplicate\s+(?:class|group|code)|extract\s+(?:shared|cohesive|submodules?|helpers?))\b",
+    re.IGNORECASE,
+)
+
+_FUNCTION_SMELL_RE = re.compile(
+    r"(?:`(?:God Function|Feature Envy|Long Method|Shotgun Surgery|Data Clump):\s*[a-zA-Z0-9_]+`|`[a-zA-Z0-9_]+`\s+with\s+CC=)",
+)
+
+
 def _extract_symbol_from_corpus(text_corpus: str) -> str | None:
     for pat in _SYMBOL_PATTERNS:
-        m_sym = re.search(pat, text_corpus)
-        if m_sym:
+        for m_sym in re.finditer(pat, text_corpus):
             candidate = m_sym.group(1).strip()
-            if candidate and not candidate.isdigit() and len(candidate) > 1:
+            if (
+                candidate
+                and not candidate.isdigit()
+                and len(candidate) > 1
+                and candidate.lower() not in _SYMBOL_STOPWORDS
+            ):
                 return candidate
     return None
 
 
+def _extract_prompt_symbols(text_corpus: str) -> list[str]:
+    """Return all non-stopword symbol candidates found in *text_corpus*."""
+    found: list[str] = []
+    for pat in _SYMBOL_PATTERNS:
+        for m_sym in re.finditer(pat, text_corpus):
+            candidate = m_sym.group(1).strip()
+            if (
+                candidate
+                and not candidate.isdigit()
+                and len(candidate) > 1
+                and candidate.lower() not in _SYMBOL_STOPWORDS
+                and candidate not in found
+            ):
+                found.append(candidate)
+    return found
+
+
+def _is_module_refactor_request(request: dict[str, Any]) -> bool:
+    if request.get("target_symbol") or request.get("target_line") is not None:
+        return False
+    text_corpus = " ".join(
+        [
+            str(request.get("prompt") or ""),
+            str(request.get("ticket_description") or ""),
+        ]
+    )
+    if not text_corpus.strip():
+        return False
+    if _FUNCTION_SMELL_RE.search(text_corpus):
+        return False
+    if _MODULE_REFACTOR_RE.search(text_corpus):
+        return True
+    if "refactor" in text_corpus.lower() and _extract_symbol_from_corpus(text_corpus) is None:
+        return True
+    return False
+
+
 def _resolve_target_symbol_and_line(rel_path: str, request: dict[str, Any]) -> _TargetRef:
     """Resolve target symbol name and line number for rel_path from request metadata."""
     if (direct := _resolve_target_from_request_meta(request)) is not None:
@@ -156,11 +247,15 @@ def _focused_file_slice(
     When the file exceeds ``LARGE_FILE_SLICE_THRESHOLD_CHARS`` and target
     symbol or line metadata is present in ``request``, a focused slice is
     generated instead of dumping the entire file; otherwise the content is
-    returned unchanged.
+    returned unchanged. Module-level split/refactor requests preserve full
+    files up to ``_MODULE_REFACTOR_MAX_CHARS`` (64,000 chars).
     """
     if not request or len(content) <= LARGE_FILE_SLICE_THRESHOLD_CHARS:
         return _FocusedSlice(content, None)
 
+    if len(content) <= _MODULE_REFACTOR_MAX_CHARS and _is_module_refactor_request(request):
+        return _FocusedSlice(content, None)
+
     target = _resolve_target_symbol_and_line(rel, request)
     if target.symbol and lang == "py":
         code_slice = _extract_python_symbol_slice(content, target.symbol)
@@ -176,3 +271,25 @@ def _focused_file_slice(
             f"focused slice around line {target.line}, lines {code_slice.start_line}-{code_slice.end_line}",
         )
     return _FocusedSlice(content, None)
+
+
+def slice_file_content(
+    rel: str,
+    content: str,
+    prompt_or_lang: str | dict[str, Any] = "",
+    request: dict[str, Any] | None = None,
+    *,
+    max_chars: int = _MODULE_REFACTOR_MAX_CHARS,
+) -> str | _FocusedSlice:
+    """Slice *content* for *rel* given either a prompt/request or (lang, request)."""
+    if request is not None:
+        return _focused_file_slice(rel, content, str(prompt_or_lang), request)
+    if isinstance(prompt_or_lang, dict):
+        req = prompt_or_lang
+    else:
+        req = {"prompt": str(prompt_or_lang)}
+    lang = "py" if str(rel).endswith(".py") else ""
+    if len(content) <= max_chars and _is_module_refactor_request(req):
+        return content
+    return _focused_file_slice(rel, content, lang, req).content
+