From cf7c5c020c30f5523f94f96bd4863a9f9e476be3 Mon Sep 17 00:00:00 2001 From: Cemberk Date: Thu, 13 Aug 2026 08:15:25 -0500 Subject: [PATCH 1/3] fix(profiling): stop losing TraceLens reports to two avoidable outcomes Both come from running the tools against real traces on an AMD GPU node. - `--short_kernel_study` makes TraceLens die while writing the workbook on some real traces: one of its short-kernel sheets has MultiIndex columns, and TraceLens writes every sheet with `index=False`, which pandas refuses. Every CSV is written before that, so the analysis itself was fine and only the .xlsx was lost, but the non-zero exit turned the trace into a FAILURE. madengine no longer requests the flag; it stays available through the analyzer's trailing extra args. Reported upstream as AMD-AGI/TraceLens#938. - A trace that holds no GPU activity is now SKIPPED rather than FAILURE, with a reason. dynolog configures every process that registered with it, so a torchrun job hands us the launcher's trace alongside the ranks': it only supervises children and runs no kernels. Every distributed run therefore ended with a failure row sitting next to its real report. The docs gain the two things only the GPU runs could show: an iteration-based request that lands before the workload's first optimizer step captures nothing, and an N-rank torchrun job produces N+1 traces. Co-authored-by: Cursor --- docs/profiling.md | 8 ++- .../scripts/common/tools/tracelens_analyze.py | 37 +++++++++++-- tests/e2e/test_tracelens_dummy_pipeline.py | 52 +++++++++++++++++++ .../TraceLens/Reporting/_dummy.py | 34 +++++++++++- tests/unit/test_tracelens_analyze.py | 42 +++++++++++++++ 5 files changed, 166 insertions(+), 7 deletions(-) diff --git a/docs/profiling.md b/docs/profiling.md index 0b7b6521..07a9c66a 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -399,7 +399,11 @@ For a short-lived workload, shorten the warmup so the request lands while the mo } ``` -**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. +**No trace produced?** The teardown script reports how many traces it found. `no PyTorch process registered` means `dyno gputrace` never matched the workload: confirm the model is PyTorch, and raise `TORCH_PROFILE_WARMUP_S` and `TORCH_PROFILE_MAX_ATTEMPTS` for slow-starting jobs. `dyno rejected the trace request` means the request itself was refused, which points at a dynolog version mismatch rather than at the workload. + +**Trace produced but empty?** Iteration-based capture waits for the workload's next `optimizer.step()`, so the request has to land after training has actually started. A request that arrives while the model is still being built (or while MIOpen is autotuning the first convolution) yields a trace with no GPU activity. The warmup must cover startup, not just process launch. + +Every process that registers with dynolog is traced, including the `torchrun` launcher, which supervises its children and runs no kernels itself. An `N`-rank job therefore produces `N + 1` traces, and the launcher's holds nothing to report. ### tracelens - TraceLens Trace Analysis @@ -416,6 +420,8 @@ Each trace format is routed to the matching TraceLens report: **Unreadable formats:** TraceLens cannot read rocprofv3's default SQLite (`*_results.db`) or RPD (`.rpd`) databases. Those are listed in the summary as `SKIPPED` with a pointer at a preset that works — use `rocprofv3_lightweight` for JSON or `rocprofv3_perfetto` for `.pftrace`. For `rpd`, point TraceLens at the `trace.json` its post-script writes alongside the database. +**Traces with nothing to analyze:** a trace that holds no GPU activity is also reported as `SKIPPED` rather than as a failure. The usual source is the `torchrun` launcher process, which dynolog traces along with the ranks that do the work. + #### Analyzing on the Host (Recommended) Running analysis on the host keeps TraceLens' pinned `protobuf` and `xprof` out of your workload image: diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index 6abc438e..d318b4cd 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -98,6 +98,14 @@ # Read size used when checking a trace for undecodable bytes and rewriting it. _SANITIZE_CHUNK_BYTES = 1 << 20 +# What TraceLens says when a trace holds no GPU activity for it to report on. +_NO_GPU_EVENTS_ERROR = "No GPU events found in the trace" +_NO_GPU_EVENTS_REASON = ( + "the trace holds no GPU activity, so there is nothing to report. dynolog " + "configures every process that registered with it, which for a torchrun job " + "includes the launcher: it only supervises its children and runs no kernels." +) + SUMMARY_CSV_FIELDS = ( "trace_file", "kind", @@ -335,6 +343,11 @@ def _failure_detail(returncode: int, output: str) -> str: return lines[-1] if lines else f"exit code {returncode}" +def _has_no_gpu_events(output: str) -> bool: + """Return True when TraceLens refused a trace for carrying no GPU activity.""" + return _NO_GPU_EVENTS_ERROR in output + + def _report_stem(path: str, root: str) -> str: """Return a filesystem-safe, collision-resistant name for a trace's reports.""" relative = os.path.relpath(path, root) @@ -348,6 +361,13 @@ def _report_stem(path: str, root: str) -> str: def _pytorch_args( trace: str, out_base: str, gpu_arch: Optional[str], extra: Sequence[str] ) -> List[str]: + # --short_kernel_study is deliberately not requested. On some real traces its + # sheets have MultiIndex columns, and TraceLens writes every sheet with + # `index=False`, which pandas refuses: "Writing to Excel with MultiIndex + # columns and no index ('index'=False) is not yet implemented". That loses the + # whole workbook after the CSVs are already written. Pass it back through the + # analyzer's trailing extra args if you want those sheets. + # https://github.com/AMD-AGI/TraceLens/issues/938 args = [ "--profile_json_path", trace, @@ -356,7 +376,6 @@ def _pytorch_args( "--output_csvs_dir", f"{out_base}_csv", "--enable_kernel_summary", - "--short_kernel_study", ] if gpu_arch: args += ["--gpu_arch_platform", gpu_arch] @@ -573,6 +592,16 @@ def analyze( for trace, kind, tool, args in jobs: print(f"[tracelens] {tool}: {trace}", flush=True) code, output = _run(_build_command(interpreter, tool, args)) + if code == 0: + status, detail, produced = "SUCCESS", "", os.path.relpath(output_dir, root) + elif _has_no_gpu_events(output): + # Nothing was wrong with the analysis, and nothing was produced. + print(f"[tracelens] skipping {trace}: {_NO_GPU_EVENTS_REASON}", flush=True) + status, detail, produced = "SKIPPED", _NO_GPU_EVENTS_REASON, "" + else: + status = "FAILURE" + detail = _failure_detail(code, output) + produced = os.path.relpath(output_dir, root) results.append( { "trace_file": ( @@ -580,9 +609,9 @@ def analyze( ), "kind": kind, "tracelens_tool": tool, - "status": "SUCCESS" if code == 0 else "FAILURE", - "output": os.path.relpath(output_dir, root), - "detail": "" if code == 0 else _failure_detail(code, output), + "status": status, + "output": produced, + "detail": detail, } ) diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py index b3da5679..27024b93 100644 --- a/tests/e2e/test_tracelens_dummy_pipeline.py +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -188,6 +188,32 @@ def write_chrome_trace(path: Path) -> Path: return path +def write_chrome_trace_without_gpu_events(path: Path) -> Path: + """Write the kind of trace dynolog collects from a torchrun launcher. + + The launcher registers with dynolog like any other PyTorch process, so it is + configured and traced alongside the ranks, but it only supervises children: + its trace carries Python frames and no GPU work at all. + """ + payload = { + "schemaVersion": 1, + "traceEvents": [ + { + "ph": "X", + "cat": "python_function", + "name": "torch/distributed/run.py(892): main", + "pid": 724, + "tid": 724, + "ts": 100, + "dur": 9000, + } + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def write_rocprof_json(path: Path) -> Path: """Write a rocprofv3 JSON result document, as ``rocprofv3_lightweight`` does. @@ -391,6 +417,32 @@ def test_a_valid_trace_is_analyzed_where_it_lies(self, profiled_run, dummy_trace assert analyzed == str(profiled_run / "rocprof_output" / "1234_results.json") assert "sanitized copy" not in result.stdout + def test_a_launcher_trace_is_skipped_rather_than_failed( + self, tmp_path, dummy_tracelens + ): + """A torchrun job hands madengine one trace with no GPU work every run. + + dynolog configures every process that registered with it, so the ranks' + traces arrive next to the launcher's. Failing on the launcher would mean + every distributed run ends with a failure row beside its real report. + """ + work = tmp_path / "torchrun" + write_chrome_trace(work / "torch_profiler_output" / "libkineto_trace_892.json") + write_chrome_trace_without_gpu_events( + work / "torch_profiler_output" / "libkineto_trace_724.json" + ) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode == 0, result.stdout + rows = {Path(row["trace_file"]).name: row for row in summary_rows(work)} + assert rows["libkineto_trace_892.json"]["status"] == "SUCCESS" + launcher = rows["libkineto_trace_724.json"] + assert launcher["status"] == "SKIPPED", launcher + assert "no GPU activity" in launcher["detail"] + # No report was written for it, so naming an output would mislead. + assert launcher["output"] == "" + def test_gzipped_kineto_trace_is_analyzed(self, tmp_path, dummy_tracelens): """tensorboard_trace_handler's gzipped traces are picked up too.""" work = tmp_path / "gz" diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py index 0f81bb00..feb423a7 100644 --- a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -18,6 +18,7 @@ import argparse import glob +import gzip import json import os import sys @@ -29,9 +30,10 @@ _SPECS: Dict[str, Dict[str, object]] = { "TraceLens_generate_perf_report_pytorch": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), - "switches": ("--enable_kernel_summary", "--short_kernel_study"), + "switches": ("--enable_kernel_summary",), "optional": ("--gpu_arch_platform",), "input_file": "profile_json_path", + "needs_gpu_events": True, }, "TraceLens_generate_perf_report_rocprof": { "required": ("--profile_json_path", "--output_xlsx_path", "--output_csvs_dir"), @@ -65,6 +67,14 @@ _FORCED_FAILURE_EXIT_CODE = 3 +# Kineto categories that carry GPU work. A trace without any of them gives +# TraceLens nothing to report on, and it says so rather than writing an empty +# report. dynolog traces every process that registered with it, so a torchrun +# job hands madengine one such trace per run: the launcher's. +_GPU_ACTIVITY_CATEGORIES = frozenset( + {"kernel", "gpu_memcpy", "gpu_memset", "gpu_user_annotation"} +) + def _parser(entry_point: str) -> argparse.ArgumentParser: spec = _SPECS[entry_point] @@ -108,6 +118,21 @@ def _check_input_file(path: str) -> Optional[int]: return None +def _check_gpu_events(path: str) -> Optional[int]: + opener = gzip.open if path.endswith(".gz") else open + try: + with opener(path, "rt", encoding="utf-8") as handle: + payload = json.load(handle) + except (OSError, ValueError): + # Unreadable input is what the other checks are for. + return None + events = payload.get("traceEvents", []) if isinstance(payload, dict) else [] + for event in events: + if isinstance(event, dict) and event.get("cat") in _GPU_ACTIVITY_CATEGORIES: + return None + return _fail("ValueError: No GPU events found in the trace") + + def _check_input_glob(pattern: str, world_size: str) -> Optional[int]: matches = [p for p in glob.glob(pattern, recursive=True) if os.path.isfile(p)] if len(matches) < 2: @@ -172,9 +197,14 @@ def _report(entry_point: str, argv: Optional[Sequence[str]]) -> int: spec = _SPECS[entry_point] if "input_file" in spec: - failure = _check_input_file(getattr(parsed, str(spec["input_file"]))) + trace = getattr(parsed, str(spec["input_file"])) + failure = _check_input_file(trace) if failure is not None: return failure + if spec.get("needs_gpu_events"): + failure = _check_gpu_events(trace) + if failure is not None: + return failure if "input_glob" in spec: failure = _check_input_glob( getattr(parsed, str(spec["input_glob"])), parsed.world_size diff --git a/tests/unit/test_tracelens_analyze.py b/tests/unit/test_tracelens_analyze.py index db73d7ae..fc684872 100644 --- a/tests/unit/test_tracelens_analyze.py +++ b/tests/unit/test_tracelens_analyze.py @@ -154,6 +154,18 @@ def test_pytorch_args_omit_roofline_without_arch(self, analyzer): args = analyzer._pytorch_args("t.json", "/out/t", None, []) assert "--gpu_arch_platform" not in args + def test_pytorch_args_do_not_request_the_short_kernel_study(self, analyzer): + # TraceLens writes that sheet with index=False, and pandas refuses when + # its columns are a MultiIndex, which loses the whole workbook. + args = analyzer._pytorch_args("t.json", "/out/t", "MI300X", []) + assert "--short_kernel_study" not in args + + def test_the_short_kernel_study_can_be_asked_for_explicitly(self, analyzer): + args = analyzer._pytorch_args( + "t.json", "/out/t", "MI300X", ["--short_kernel_study"] + ) + assert "--short_kernel_study" in args + def test_pftrace_produces_three_complementary_reports(self, analyzer): jobs = analyzer._pftrace_jobs("t.pftrace", "/out/t", []) assert [tool for tool, _ in jobs] == [ @@ -254,6 +266,36 @@ def fake_run(command, cwd=None): assert len(rows) == 11 assert set(rows[0]) == set(analyzer.SUMMARY_CSV_FIELDS) + def test_a_trace_without_gpu_activity_is_skipped_not_failed( + self, analyzer, tmp_path, monkeypatch + ): + """dynolog traces the torchrun launcher too, and it runs no kernels. + + Reporting that as a failure means every multi-process run profiled + through dynolog ends with a failure row next to its real report. + """ + _write(tmp_path, "torch_profiler_output/libkineto_trace_724.json", CHROME_TRACE) + monkeypatch.setattr( + analyzer, + "_run", + lambda command, cwd=None: ( + 1, + "Traceback (most recent call last):\n" + "ValueError: No GPU events found in the trace", + ), + ) + + summary = analyzer.analyze( + root=str(tmp_path), output_dir=str(tmp_path / "out"), python=sys.executable + ) + assert summary["failed"] == 0 + assert summary["skipped"] == 1 + row = summary["results"][0] + assert row["status"] == "SKIPPED" + assert "no GPU activity" in row["detail"] + # Nothing was written, so pointing at a report directory would mislead. + assert row["output"] == "" + def test_records_failure_detail(self, analyzer, tmp_path, monkeypatch): _write(tmp_path, "rocprof_output/1_results.json", b"{}") monkeypatch.setattr( From e560b32ef4d9dbbc8b8ca95b87f0f73dd8d4e7c1 Mon Sep 17 00:00:00 2001 From: Cemberk Date: Thu, 13 Aug 2026 10:01:23 -0500 Subject: [PATCH 2/3] docs(profiling): record why rocprof reports have no workbook TraceLens' rocprof report writes either the CSVs or the workbook, never both, and madengine asks for the CSVs. That also makes the flag we had to drop for the PyTorch report harmless here, which is worth saying next to the arguments so nobody drops it twice. Co-authored-by: Cursor --- docs/profiling.md | 2 ++ src/madengine/scripts/common/tools/tracelens_analyze.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/docs/profiling.md b/docs/profiling.md index 07a9c66a..60847733 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -478,6 +478,8 @@ Use a mode-specific variant to restrict analysis to one trace kind: | `tracelens_pftrace` | Perfetto traces only | | `tracelens_collective` | Multi-rank collective report only | +Kineto traces yield both an `.xlsx` workbook and a CSV directory per trace. rocprofv3 traces yield the CSV directory only: TraceLens' rocprof report writes either format but not both, and madengine asks for the CSVs. + **Full PyTorch pipeline** — capture and analyze in one run: ```bash diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index d318b4cd..41f03cc6 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -383,6 +383,10 @@ def _pytorch_args( def _rocprof_args(trace: str, out_base: str, extra: Sequence[str]) -> List[str]: + # --short_kernel_study is safe to keep here, unlike for the PyTorch report: + # this generator writes either the CSVs or the workbook, never both, so with + # a CSV directory requested it never reaches the Excel writer that the flag's + # MultiIndex sheets break. That also means --output_xlsx_path is ignored. return [ "--profile_json_path", trace, From 066844ff726da85ec6e5165f377d91b64faee833 Mon Sep 17 00:00:00 2001 From: Stephen Shao Date: Fri, 2 Oct 2026 15:25:20 +0000 Subject: [PATCH 3/3] fix(profiling): tighten TraceLens dummy and wording from review - Dummy TraceLens fails on a trace it cannot read (corrupt or truncated gzip, invalid JSON) instead of passing or crashing. - --short_kernel_study guidance: pass it with --mode pytorch, since extra args reach every TraceLens command. - The no-GPU-events reason names the dynolog launcher as one common cause rather than the only one. - Fix a docs typo. Co-Authored-By: Claude --- docs/profiling.md | 2 +- .../scripts/common/tools/tracelens_analyze.py | 12 ++++++---- tests/e2e/test_tracelens_dummy_pipeline.py | 24 +++++++++++++++++++ .../TraceLens/Reporting/_dummy.py | 6 ++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/docs/profiling.md b/docs/profiling.md index 60847733..179af6d1 100644 --- a/docs/profiling.md +++ b/docs/profiling.md @@ -403,7 +403,7 @@ For a short-lived workload, shorten the warmup so the request lands while the mo **Trace produced but empty?** Iteration-based capture waits for the workload's next `optimizer.step()`, so the request has to land after training has actually started. A request that arrives while the model is still being built (or while MIOpen is autotuning the first convolution) yields a trace with no GPU activity. The warmup must cover startup, not just process launch. -Every process that registers with dynolog is traced, including the `torchrun` launcher, which supervises its children and runs no kernels itself. An `N`-rank job therefore produces `N + 1` traces, and the launcher's holds nothing to report. +Every process that registers with dynolog is traced, including the `torchrun` launcher, which supervises its children and runs no kernels itself. An `N`-rank job therefore produces `N + 1` traces, and the launcher holds nothing to report. ### tracelens - TraceLens Trace Analysis diff --git a/src/madengine/scripts/common/tools/tracelens_analyze.py b/src/madengine/scripts/common/tools/tracelens_analyze.py index e3d4fb71..1e5ca4da 100644 --- a/src/madengine/scripts/common/tools/tracelens_analyze.py +++ b/src/madengine/scripts/common/tools/tracelens_analyze.py @@ -101,9 +101,10 @@ # What TraceLens says when a trace holds no GPU activity for it to report on. _NO_GPU_EVENTS_ERROR = "No GPU events found in the trace" _NO_GPU_EVENTS_REASON = ( - "the trace holds no GPU activity, so there is nothing to report. dynolog " - "configures every process that registered with it, which for a torchrun job " - "includes the launcher: it only supervises its children and runs no kernels." + "the trace holds no GPU activity, so there is nothing to report. One common " + "cause: dynolog configures every process that registered with it, which for " + "a torchrun job includes the launcher, and the launcher only supervises its " + "children and runs no kernels." ) SUMMARY_CSV_FIELDS = ( @@ -401,8 +402,9 @@ def _pytorch_args( # sheets have MultiIndex columns, and TraceLens writes every sheet with # `index=False`, which pandas refuses: "Writing to Excel with MultiIndex # columns and no index ('index'=False) is not yet implemented". That loses the - # whole workbook after the CSVs are already written. Pass it back through the - # analyzer's trailing extra args if you want those sheets. + # whole workbook after the CSVs are already written. To get those sheets, pass + # it through the trailing extra args with `--mode pytorch`: in other modes the + # extra args reach every TraceLens command, including ones that reject it. # https://github.com/AMD-AGI/TraceLens/issues/938 args = [ "--profile_json_path", diff --git a/tests/e2e/test_tracelens_dummy_pipeline.py b/tests/e2e/test_tracelens_dummy_pipeline.py index 9efb0960..f00f336d 100644 --- a/tests/e2e/test_tracelens_dummy_pipeline.py +++ b/tests/e2e/test_tracelens_dummy_pipeline.py @@ -548,6 +548,30 @@ def test_module_fallback_still_reports_failures( assert [r["status"] for r in pytorch_rows] == ["FAILURE"] assert "forced failure" in pytorch_rows[0]["detail"] + @pytest.mark.parametrize( + "payload", + [ + b"not gzip data", + gzip.compress(b'{"traceEvents": [{"cat": "kernel"}]}')[:-8], + gzip.compress(b'{"traceEvents": ['), + ], + ids=["not-gzip", "truncated-gzip", "invalid-json"], + ) + def test_unreadable_gzipped_trace_fails_like_real_tracelens( + self, tmp_path, dummy_tracelens, payload + ): + work = tmp_path / "corrupt" + trace = work / "torch_profiler_output" / "libkineto_trace_1.json.gz" + trace.parent.mkdir(parents=True) + trace.write_bytes(payload) + + result = run_analyzer(work, dummy_tracelens) + + assert result.returncode != 0, result.stdout + rows = [r for r in summary_rows(work) if r["kind"] == "pytorch"] + assert [r["status"] for r in rows] == ["FAILURE"] + assert "dummy TraceLens: could not read" in rows[0]["detail"] + def test_one_failed_report_does_not_hide_the_others( self, profiled_run, dummy_tracelens ): diff --git a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py index feb423a7..5e0a2757 100644 --- a/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py +++ b/tests/fixtures/dummy_tracelens/TraceLens/Reporting/_dummy.py @@ -123,9 +123,9 @@ def _check_gpu_events(path: str) -> Optional[int]: try: with opener(path, "rt", encoding="utf-8") as handle: payload = json.load(handle) - except (OSError, ValueError): - # Unreadable input is what the other checks are for. - return None + except (OSError, EOFError, ValueError) as exc: + # Real TraceLens fails on a trace it cannot read, gzipped or not. + return _fail(f"could not read {path}: {type(exc).__name__}: {exc}") events = payload.get("traceEvents", []) if isinstance(payload, dict) else [] for event in events: if isinstance(event, dict) and event.get("cat") in _GPU_ACTIVITY_CATEGORIES: