diff --git a/.gitignore b/.gitignore index f56ab36d2..c3b4d4107 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ orig/*/* # Build files build/ +build_dwarf/ .ninja_* build.ninja diff --git a/configure.py b/configure.py index 1726413f0..fb13aa366 100755 --- a/configure.py +++ b/configure.py @@ -145,6 +145,17 @@ choices=["all", "off", "error"], help="how to handle warnings", ) +parser.add_argument( + "--dwarf", + action="store_true", + help="add DWARF debug info (-sym on) to every object, without -DDEBUG=1", +) +parser.add_argument( + "--link-all", + dest="link_all", + action="store_true", + help="link every object that has source, including NonMatching ones (implies --non-matching)", +) parser.add_argument( "--no-progress", dest="progress", @@ -164,7 +175,7 @@ config.binutils_path = args.binutils config.compilers_path = args.compilers config.generate_map = args.map -config.non_matching = args.non_matching +config.non_matching = args.non_matching or args.link_all config.sjiswrap_path = args.sjiswrap config.ninja_path = args.ninja config.progress = args.progress @@ -189,14 +200,14 @@ "-mgekko", "--strip-local-absolute", "-I include", - f"-I build/{config.version}/include", + f"-I {config.out_path().as_posix()}/include", f"--defsym version={version_num}", ] config.ldflags = [ "-fp hardware", "-nodefaults", ] -if args.debug: +if args.debug or args.dwarf: config.ldflags.append("-g") # Or -gdwarf-2 for Wii linkers if args.map: config.ldflags.append("-mapunused") @@ -234,7 +245,7 @@ "-i include", "-i extern/sdk/include", "-i extern/sdk/libc", - f"-i build/{config.version}/include", + f"-i {config.out_path().as_posix()}/include", f"-DVERSION={version_num}", f"-DRSTL_VERSION={RSTL_VERSIONS[config.version]}", "-DPRIME1", @@ -292,7 +303,7 @@ "-i extern/sdk/include", "-i extern/sdk/libc", "-i extern/zlib-1.1.3", - f"-i build/{config.version}/include", + f"-i {config.out_path().as_posix()}/include", f"-DVERSION={version_num}", f"-DRSTL_VERSION={RSTL_VERSIONS[config.version]}", "-DPRIME1", @@ -3248,6 +3259,20 @@ def EquivalentFor(*versions): config.libs.append(Rel(module, objects)) +if args.dwarf: + # Per object, since not every object's cflags derive from cflags_base. + for lib in config.libs: + for obj in lib["objects"]: + obj.options["extra_cflags"] = [*obj.options["extra_cflags"], "-sym on"] + +if args.link_all: + for lib in config.libs: + for obj in lib["objects"]: + src_dir = obj.options["src_dir"] or lib.get("src_dir") or config.src_dir + if (Path(src_dir) / obj.options["source"]).exists(): + obj.completed = True + + # Optional callback to adjust link order. This can be used to add, remove, or reorder objects. # This is called once per module, with the module ID and the current link order. # diff --git a/scripts/export_data_symbols.py b/scripts/export_data_symbols.py new file mode 100644 index 000000000..293df79b7 --- /dev/null +++ b/scripts/export_data_symbols.py @@ -0,0 +1,80 @@ +"""Exports named data symbols from config//symbols.txt, one file per version. + +Keeps every named data object (vtables, globals, statics) and drops functions and +dtk's auto-generated names (lbl_*, @NNN, ...). Output lines are `0xADDRESS name`, +sorted by address. + +--aliases adds extra names for existing symbols (e.g. for ones that are unnamed, or +whose name differs between versions). Each non-comment line of that file is +` `. + +Usage: python scripts/export_data_symbols.py -o OUT_DIR [--aliases FILE] VERSION... +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import List, Optional, Tuple + +from symbols import parse_symbols_file + +ROOT = Path(__file__).resolve().parent.parent +AUTO_NAME = re.compile(r"^(lbl_|@|jumptable_|gap_|\.\.\.)") + + +def load_aliases(path: Path, version: str) -> List[Tuple[str, str]]: + aliases = [] + for line in path.read_text().splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + alias_version, name, target = line.split() + if alias_version == version: + aliases.append((name, target)) + return aliases + + +def export(version: str, out_dir: Path, aliases_path: Optional[Path]) -> None: + symbols = parse_symbols_file(ROOT / "config" / version / "symbols.txt") + by_name = {s.name: s.address for s in symbols} + kept = [ + (s.name, s.address) + for s in symbols + if s.attrs.get("type") == "object" and s.section != ".text" and not AUTO_NAME.match(s.name) + ] + + if aliases_path is not None: + names = {name for name, _ in kept} + for name, target in load_aliases(aliases_path, version): + if target not in by_name: + sys.exit(f"{version}: alias {name} -> {target}: no such symbol") + if name in names: + sys.exit(f"{version}: alias {name} is already a symbol; drop the alias") + kept.append((name, by_name[target])) + + kept.sort(key=lambda s: (s[1], s[0])) + out = out_dir / f"{version}.txt" + with out.open("w") as f: + f.write(f"# Generated by prime-decomp scripts/export_data_symbols.py from config/{version}/symbols.txt\n") + for name, address in kept: + f.write(f"0x{address:08X} {name}\n") + print(f"wrote {out} ({len(kept)} symbols)") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("-o", "--out-dir", type=Path, required=True, help="writes /.txt") + parser.add_argument("--aliases", type=Path, help="extra names to add (see above)") + parser.add_argument("versions", nargs="+", type=str.upper, metavar="VERSION") + args = parser.parse_args() + + args.out_dir.mkdir(parents=True, exist_ok=True) + for version in args.versions: + export(version, args.out_dir, args.aliases) + + +if __name__ == "__main__": + main() diff --git a/scripts/export_types.py b/scripts/export_types.py new file mode 100644 index 000000000..11a511324 --- /dev/null +++ b/scripts/export_types.py @@ -0,0 +1,135 @@ +"""Exports every struct, union and enum layout as gzipped JSON, one file per version. + +For each version, builds main.elf with `configure.py --dwarf` into a separate +build directory, then runs `dtk dwarf types` on it. Versions in LINK_ALL_VERSIONS +also get `--link-all`, so the export covers NonMatching units too; the rest use +`--non-matching` until they link that way. Member offsets come from the headers, +so they're right even where the code isn't matching yet. +Addresses in this ELF are NOT; take those from config//symbols.txt +(see export_data_symbols.py). + +Versions without files in orig// are skipped. build.ninja, objdiff.json +and compile_commands.json are restored afterwards. + +Usage: python scripts/export_types.py -o OUT_DIR [-v VERSION ...] [-- CONFIGURE_ARGS...] + e.g. python scripts/export_types.py -o types -- --dtk ../decomp-toolkit +""" + +from __future__ import annotations + +import argparse +import gzip +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import List + +ROOT = Path(__file__).resolve().parent.parent +GENERATED = ["build.ninja", "objdiff.json", "compile_commands.json"] +# Versions whose NonMatching objects all compile and link. +LINK_ALL_VERSIONS = {"GM8E01_00", "GM8E01_01"} +LINK_NONMATCHING_VERSIONS = {"GM8P01_00"} +GAMECUBE_VERSIONS = ["GM8E01_00", "GM8E01_01", "GM8E01_48", "GM8P01_00", "GM8J01_00", "GM8E01_02"] + + +def has_orig(version: str) -> bool: + orig = ROOT / "orig" / version + return orig.is_dir() and any(p.name != ".gitkeep" for p in orig.iterdir()) + + +def dtk_binary(build_dir: Path, configure_args: List[str]) -> Path: + # Mirrors how tools/project.py resolves --dtk. + exe = ".exe" if sys.platform == "win32" else "" + if "--dtk" in configure_args: + dtk = ROOT / configure_args[configure_args.index("--dtk") + 1] + if dtk.is_file(): + return dtk + return ROOT / build_dir / "tools" / "release" / f"dtk{exe}" + return ROOT / build_dir / "tools" / f"dtk{exe}" + + +def export(version: str, build_dir: Path, out_dir: Path, configure_args: List[str]) -> None: + export_args = [ + sys.executable, "configure.py", + "--version", version, + "--build-dir", str(build_dir), + "--dwarf", "--no-progress", + ] + + if version in LINK_NONMATCHING_VERSIONS: + export_args.append("--non-matching") + if version in LINK_ALL_VERSIONS: + export_args.append("--link-all") + + export_args.extend(configure_args) + + subprocess.run( + export_args, + cwd=ROOT, + check=True, + ) + elf = build_dir / version / "main.elf" + subprocess.run(["ninja", str(elf)], cwd=ROOT, check=True) + + dtk = dtk_binary(build_dir, configure_args) + with tempfile.TemporaryDirectory() as tmp: + # dtk logs to stdout, so write through a file rather than a pipe. + json_path = Path(tmp) / "types.json" + subprocess.run([str(dtk), "dwarf", "types", str(elf), "-o", str(json_path)], cwd=ROOT, check=True) + out = out_dir / f"{version}.json.gz" + tmp_out = out.with_suffix(".gz.tmp") + # mtime=0 so regenerating from the same ELF is byte-identical. + with open(tmp_out, "wb") as raw, gzip.GzipFile(fileobj=raw, mode="wb", compresslevel=9, mtime=0) as gz: + gz.write(json_path.read_bytes()) + tmp_out.replace(out) + print(f"wrote {out}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("-o", "--out-dir", type=Path, required=True, help="writes /.json.gz") + parser.add_argument("-v", "--version", dest="versions", action="append", type=str.upper, + help="version to export (repeatable; default: every GameCube version)") + parser.add_argument("--build-dir", type=Path, default=Path("build_dwarf"), + help="build directory, relative to the repo root (default: build_dwarf)") + parser.add_argument("configure_args", nargs="*", help="extra configure.py arguments, after --") + args = parser.parse_args() + + versions = args.versions or GAMECUBE_VERSIONS + out_dir = args.out_dir.resolve() + out_dir.mkdir(parents=True, exist_ok=True) + + saved = tempfile.mkdtemp() + for name in GENERATED: + if (ROOT / name).exists(): + shutil.copy2(ROOT / name, saved) + done, skipped, failed = [], [], [] + try: + for version in versions: + if not has_orig(version): + skipped.append(version) + continue + print(f"== {version}") + try: + export(version, args.build_dir, out_dir, args.configure_args) + done.append(version) + except subprocess.CalledProcessError: + failed.append(version) + finally: + for name in GENERATED: + if (Path(saved) / name).exists(): + shutil.copy2(Path(saved) / name, ROOT / name) + shutil.rmtree(saved) + + print() + print(f"exported: {' '.join(done) or 'none'}") + if skipped: + print(f"skipped (nothing in orig//): {' '.join(skipped)}") + if failed: + sys.exit(f"FAILED: {' '.join(failed)}") + + +if __name__ == "__main__": + main()