Add scripts/configuration for exporting DWARF symbols in json format

.gitignore

@@ -14,6 +14,7 @@ orig/*/*
 
 # Build files
 build/
+build_dwarf/
 .ninja_*
 build.ninja
 

configure.py

@@ -145,6 +145,17 @@
     choices=["all", "off", "error"],
     help="how to handle warnings",
 )
+parser.add_argument(
+    "--dwarf",
+    action="store_true",
+    help="add DWARF debug info (-sym on) to every object, without -DDEBUG=1",
+)
+parser.add_argument(
+    "--link-all",
+    dest="link_all",
+    action="store_true",
+    help="link every object that has source, including NonMatching ones (implies --non-matching)",
+)
 parser.add_argument(
     "--no-progress",
     dest="progress",
@@ -164,7 +175,7 @@
 config.binutils_path = args.binutils
 config.compilers_path = args.compilers
 config.generate_map = args.map
-config.non_matching = args.non_matching
+config.non_matching = args.non_matching or args.link_all
 config.sjiswrap_path = args.sjiswrap
 config.ninja_path = args.ninja
 config.progress = args.progress
@@ -189,14 +200,14 @@
     "-mgekko",
     "--strip-local-absolute",
     "-I include",
-    f"-I build/{config.version}/include",
+    f"-I {config.out_path().as_posix()}/include",
     f"--defsym version={version_num}",
 ]
 config.ldflags = [
     "-fp hardware",
     "-nodefaults",
 ]
-if args.debug:
+if args.debug or args.dwarf:
     config.ldflags.append("-g")  # Or -gdwarf-2 for Wii linkers
 if args.map:
     config.ldflags.append("-mapunused")
@@ -234,7 +245,7 @@
     "-i include",
     "-i extern/sdk/include",
     "-i extern/sdk/libc",
-    f"-i build/{config.version}/include",
+    f"-i {config.out_path().as_posix()}/include",
     f"-DVERSION={version_num}",
     f"-DRSTL_VERSION={RSTL_VERSIONS[config.version]}",
     "-DPRIME1",
@@ -292,7 +303,7 @@
     "-i extern/sdk/include",
     "-i extern/sdk/libc",
     "-i extern/zlib-1.1.3",
-    f"-i build/{config.version}/include",
+    f"-i {config.out_path().as_posix()}/include",
     f"-DVERSION={version_num}",
     f"-DRSTL_VERSION={RSTL_VERSIONS[config.version]}",
     "-DPRIME1",
@@ -3248,6 +3259,20 @@ def EquivalentFor(*versions):
         config.libs.append(Rel(module, objects))
 
 
+if args.dwarf:
+    # Per object, since not every object's cflags derive from cflags_base.
+    for lib in config.libs:
+        for obj in lib["objects"]:
+            obj.options["extra_cflags"] = [*obj.options["extra_cflags"], "-sym on"]
+
+if args.link_all:
+    for lib in config.libs:
+        for obj in lib["objects"]:
+            src_dir = obj.options["src_dir"] or lib.get("src_dir") or config.src_dir
+            if (Path(src_dir) / obj.options["source"]).exists():
+                obj.completed = True
+
+
 # Optional callback to adjust link order. This can be used to add, remove, or reorder objects.
 # This is called once per module, with the module ID and the current link order.
 #

scripts/export_data_symbols.py

@@ -0,0 +1,80 @@
+"""Exports named data symbols from config/<VERSION>/symbols.txt, one file per version.
+
+Keeps every named data object (vtables, globals, statics) and drops functions and
+dtk's auto-generated names (lbl_*, @NNN, ...). Output lines are `0xADDRESS name`,
+sorted by address.
+
+--aliases adds extra names for existing symbols (e.g. for ones that are unnamed, or
+whose name differs between versions). Each non-comment line of that file is
+`<VERSION> <new name> <existing symbol>`.
+
+Usage: python scripts/export_data_symbols.py -o OUT_DIR [--aliases FILE] VERSION...
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from symbols import parse_symbols_file
+
+ROOT = Path(__file__).resolve().parent.parent
+AUTO_NAME = re.compile(r"^(lbl_|@|jumptable_|gap_|\.\.\.)")
+
+
+def load_aliases(path: Path, version: str) -> List[Tuple[str, str]]:
+    aliases = []
+    for line in path.read_text().splitlines():
+        line = line.split("#", 1)[0].strip()
+        if not line:
+            continue
+        alias_version, name, target = line.split()
+        if alias_version == version:
+            aliases.append((name, target))
+    return aliases
+
+
+def export(version: str, out_dir: Path, aliases_path: Optional[Path]) -> None:
+    symbols = parse_symbols_file(ROOT / "config" / version / "symbols.txt")
+    by_name = {s.name: s.address for s in symbols}
+    kept = [
+        (s.name, s.address)
+        for s in symbols
+        if s.attrs.get("type") == "object" and s.section != ".text" and not AUTO_NAME.match(s.name)
+    ]
+
+    if aliases_path is not None:
+        names = {name for name, _ in kept}
+        for name, target in load_aliases(aliases_path, version):
+            if target not in by_name:
+                sys.exit(f"{version}: alias {name} -> {target}: no such symbol")
+            if name in names:
+                sys.exit(f"{version}: alias {name} is already a symbol; drop the alias")
+            kept.append((name, by_name[target]))
+
+    kept.sort(key=lambda s: (s[1], s[0]))
+    out = out_dir / f"{version}.txt"
+    with out.open("w") as f:
+        f.write(f"# Generated by prime-decomp scripts/export_data_symbols.py from config/{version}/symbols.txt\n")
+        for name, address in kept:
+            f.write(f"0x{address:08X} {name}\n")
+    print(f"wrote {out} ({len(kept)} symbols)")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("-o", "--out-dir", type=Path, required=True, help="writes <OUT_DIR>/<VERSION>.txt")
+    parser.add_argument("--aliases", type=Path, help="extra names to add (see above)")
+    parser.add_argument("versions", nargs="+", type=str.upper, metavar="VERSION")
+    args = parser.parse_args()
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    for version in args.versions:
+        export(version, args.out_dir, args.aliases)
+
+
+if __name__ == "__main__":
+    main()

scripts/export_types.py

@@ -0,0 +1,135 @@
+"""Exports every struct, union and enum layout as gzipped JSON, one file per version.
+
+For each version, builds main.elf with `configure.py --dwarf` into a separate
+build directory, then runs `dtk dwarf types` on it. Versions in LINK_ALL_VERSIONS
+also get `--link-all`, so the export covers NonMatching units too; the rest use
+`--non-matching` until they link that way. Member offsets come from the headers,
+so they're right even where the code isn't matching yet.
+Addresses in this ELF are NOT; take those from config/<version>/symbols.txt
+(see export_data_symbols.py).
+
+Versions without files in orig/<version>/ are skipped. build.ninja, objdiff.json
+and compile_commands.json are restored afterwards.
+
+Usage: python scripts/export_types.py -o OUT_DIR [-v VERSION ...] [-- CONFIGURE_ARGS...]
+  e.g. python scripts/export_types.py -o types -- --dtk ../decomp-toolkit
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import List
+
+ROOT = Path(__file__).resolve().parent.parent
+GENERATED = ["build.ninja", "objdiff.json", "compile_commands.json"]
+# Versions whose NonMatching objects all compile and link.
+LINK_ALL_VERSIONS = {"GM8E01_00", "GM8E01_01"}
+LINK_NONMATCHING_VERSIONS = {"GM8P01_00"}
+GAMECUBE_VERSIONS = ["GM8E01_00", "GM8E01_01", "GM8E01_48", "GM8P01_00", "GM8J01_00", "GM8E01_02"]
+
+
+def has_orig(version: str) -> bool:
+    orig = ROOT / "orig" / version
+    return orig.is_dir() and any(p.name != ".gitkeep" for p in orig.iterdir())
+
+
+def dtk_binary(build_dir: Path, configure_args: List[str]) -> Path:
+    # Mirrors how tools/project.py resolves --dtk.
+    exe = ".exe" if sys.platform == "win32" else ""
+    if "--dtk" in configure_args:
+        dtk = ROOT / configure_args[configure_args.index("--dtk") + 1]
+        if dtk.is_file():
+            return dtk
+        return ROOT / build_dir / "tools" / "release" / f"dtk{exe}"
+    return ROOT / build_dir / "tools" / f"dtk{exe}"
+
+
+def export(version: str, build_dir: Path, out_dir: Path, configure_args: List[str]) -> None:
+    export_args = [
+        sys.executable, "configure.py",
+        "--version", version,
+        "--build-dir", str(build_dir),
+        "--dwarf", "--no-progress",
+    ]
+
+    if version in LINK_NONMATCHING_VERSIONS:
+        export_args.append("--non-matching")
+    if version in LINK_ALL_VERSIONS:
+        export_args.append("--link-all")
+
+    export_args.extend(configure_args)
+
+    subprocess.run(
+        export_args,
+        cwd=ROOT,
+        check=True,
+    )
+    elf = build_dir / version / "main.elf"
+    subprocess.run(["ninja", str(elf)], cwd=ROOT, check=True)
+
+    dtk = dtk_binary(build_dir, configure_args)
+    with tempfile.TemporaryDirectory() as tmp:
+        # dtk logs to stdout, so write through a file rather than a pipe.
+        json_path = Path(tmp) / "types.json"
+        subprocess.run([str(dtk), "dwarf", "types", str(elf), "-o", str(json_path)], cwd=ROOT, check=True)
+        out = out_dir / f"{version}.json.gz"
+        tmp_out = out.with_suffix(".gz.tmp")
+        # mtime=0 so regenerating from the same ELF is byte-identical.
+        with open(tmp_out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=9, mtime=0) as gz:
+            gz.write(json_path.read_bytes())
+        tmp_out.replace(out)
+    print(f"wrote {out}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("-o", "--out-dir", type=Path, required=True, help="writes <OUT_DIR>/<VERSION>.json.gz")
+    parser.add_argument("-v", "--version", dest="versions", action="append", type=str.upper,
+                        help="version to export (repeatable; default: every GameCube version)")
+    parser.add_argument("--build-dir", type=Path, default=Path("build_dwarf"),
+                        help="build directory, relative to the repo root (default: build_dwarf)")
+    parser.add_argument("configure_args", nargs="*", help="extra configure.py arguments, after --")
+    args = parser.parse_args()
+
+    versions = args.versions or GAMECUBE_VERSIONS
+    out_dir = args.out_dir.resolve()
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    saved = tempfile.mkdtemp()
+    for name in GENERATED:
+        if (ROOT / name).exists():
+            shutil.copy2(ROOT / name, saved)
+    done, skipped, failed = [], [], []
+    try:
+        for version in versions:
+            if not has_orig(version):
+                skipped.append(version)
+                continue
+            print(f"== {version}")
+            try:
+                export(version, args.build_dir, out_dir, args.configure_args)
+                done.append(version)
+            except subprocess.CalledProcessError:
+                failed.append(version)
+    finally:
+        for name in GENERATED:
+            if (Path(saved) / name).exists():
+                shutil.copy2(Path(saved) / name, ROOT / name)
+        shutil.rmtree(saved)
+
+    print()
+    print(f"exported: {' '.join(done) or 'none'}")
+    if skipped:
+        print(f"skipped (nothing in orig/<VERSION>/): {' '.join(skipped)}")
+    if failed:
+        sys.exit(f"FAILED: {' '.join(failed)}")
+
+
+if __name__ == "__main__":
+    main()