[Profiler] Add Proton CUDA graph attribution for MRV2 (#51084)

Signed-off-by: Luossu <zhluosuu@outlook.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
鐘天楽
2026-09-14 19:02:07 +00:00
committed by GitHub
co-authored by Nick Hill
parent 5372e72a98
commit 0a5747d410
8 changed files with 657 additions and 45 deletions
+394 -6
View File
@@ -350,17 +350,24 @@ def test_profiler_entered_during_capture():
def make_proton(session_id: int | None = 7):
data = SimpleNamespace(
advance_phase=Mock(side_effect=range(1, 100)),
clear=Mock(),
get=Mock(return_value={"traceEvents": []}),
get_msgpack=Mock(return_value=b"profile"),
)
return SimpleNamespace(
start=Mock(return_value=session_id),
activate=Mock(),
deactivate=Mock(),
finalize=Mock(),
scope=Mock(return_value=nullcontext()),
data=data,
)
def make_proton_wrapper(
tmp_path, proton=None, triton_version="3.6.0", **config_overrides
tmp_path, proton=None, triton_version="3.7.0", **config_overrides
):
proton = proton or make_proton()
config = ProfilerConfig(
@@ -415,6 +422,13 @@ class TestProtonConfig:
},
"requires proton_data",
),
(
{
"proton_data": "trace",
"proton_graph_attribution": True,
},
"requires proton_data='tree'",
),
],
)
def test_rejects_invalid_option_combinations(self, tmp_path, options, message):
@@ -450,6 +464,7 @@ class TestProtonConfig:
("proton_mode", "pcsampling"),
("proton_hook", "triton"),
("proton_output_format", "chrome_trace"),
("proton_graph_attribution", True),
],
)
def test_rejects_proton_options_for_other_profilers(self, field, value):
@@ -478,15 +493,101 @@ class TestProtonConfig:
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.NONE),
)
def test_rejects_proton_when_cuda_graphs_are_enabled(self, tmp_path):
with pytest.raises(ValueError, match="requires CUDA graphs to be disabled"):
@pytest.mark.parametrize("encoder_only", [False, True])
@pytest.mark.parametrize("attribution", [False, True])
def test_cuda_graphs_require_attribution(self, tmp_path, encoder_only, attribution):
# Encoder graphs are independent of the decoder's cudagraph_mode.
expected = (
nullcontext()
if attribution
else pytest.raises(
ValueError, match="requires proton_graph_attribution=True"
)
)
with expected:
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=attribution,
),
compilation_config=CompilationConfig(
cudagraph_mode=(
CUDAGraphMode.NONE if encoder_only else CUDAGraphMode.FULL
),
cudagraph_mm_encoder=encoder_only,
),
)
def test_validates_default_cuda_graph_mode_after_resolution(self, tmp_path):
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton", proton_profiler_dir=str(tmp_path)
),
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.PIECEWISE
)
@pytest.mark.parametrize(
"mode",
[
"periodic_flushing",
"periodic_flushing:format=hatchet",
"PERIODIC_FLUSHING:format=hatchet",
],
)
def test_graph_attribution_rejects_periodic_flushing(self, tmp_path, mode):
# Reject before Proton's native phase manager can abort the worker.
with pytest.raises(ValueError, match="incompatible with periodic_flushing"):
ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
proton_mode=mode,
)
@pytest.mark.parametrize("attribution", [False, True])
@pytest.mark.parametrize(
"mode", ["pcsampling", "pcsampling:interval=100", "PcSampling:interval=100"]
)
@pytest.mark.parametrize("encoder_only", [False, True])
def test_pcsampling_requires_graphs_disabled(
self, tmp_path, attribution, mode, encoder_only
):
with pytest.raises(ValueError, match="PC sampling requires CUDA graphs"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=attribution,
proton_mode=mode,
),
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.NONE
if encoder_only
else CUDAGraphMode.FULL,
cudagraph_mm_encoder=encoder_only,
),
)
def test_ordinary_proton_keeps_mrv1_graph_restriction(self, tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton", proton_profiler_dir=str(tmp_path)
),
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL),
)
def test_graph_attribution_rejects_mrv1(self, tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
with pytest.raises(ValueError, match="requires the V2 model runner"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
)
)
@@ -564,7 +665,20 @@ class TestProtonProfilerWrapper:
)
def test_newer_features_reject_triton_3_6(self, tmp_path, option, value, feature):
with pytest.raises(RuntimeError, match=feature):
make_proton_wrapper(tmp_path, **{option: value})
make_proton_wrapper(tmp_path, triton_version="3.6.0", **{option: value})
@pytest.mark.parametrize("version", ["3.6.0", "unknown"])
def test_graph_attribution_requires_phase_api(self, tmp_path, version):
with pytest.raises(RuntimeError, match="requires Triton >= 3.7"):
make_proton_wrapper(
tmp_path, triton_version=version, proton_graph_attribution=True
)
def test_ordinary_profiling_still_supports_triton_3_6(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, triton_version="3.6.0")
wrapper.start()
wrapper.stop()
proton.finalize.assert_called_once_with(session=7)
@pytest.mark.parametrize(
("option", "value"),
@@ -583,6 +697,125 @@ class TestProtonProfilerWrapper:
with pytest.raises(RuntimeError, match="does not support selecting"):
make_proton_wrapper(tmp_path, proton, proton_output_format="hatchet")
def test_cuda_graph_tree_phase_is_written_at_stop(self, tmp_path):
wrapper, proton = make_proton_wrapper(
tmp_path,
proton_context="python",
proton_graph_attribution=True,
)
with wrapper.capture_cuda_graphs():
proton.start.assert_called_once()
proton.deactivate.assert_not_called()
capture_args = proton.start.call_args.kwargs
assert capture_args["context"] == "python"
assert capture_args["data"] == "tree"
proton.data.advance_phase.assert_called_once_with(7)
proton.deactivate.assert_called_once_with(session=7, flushing=True)
proton.data.clear.assert_called_once_with(7, 0)
wrapper.start()
wrapper.stop()
wrapper.start()
wrapper.stop()
assert proton.data.get.call_args_list == [call(7, 1), call(7, 2)]
proton.data.clear.assert_has_calls([call(7, 0), call(7, 1), call(7, 2)])
output_names = sorted(tmp_path.glob("proton_rank_3_*.hatchet"))
assert len(output_names) == 2
assert output_names[0].name.endswith("_run0.hatchet")
assert output_names[1].name.endswith("_run1.hatchet")
def test_capture_is_noop_without_opt_in(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path)
with wrapper.capture_cuda_graphs():
pass
proton.start.assert_not_called()
@pytest.mark.parametrize("delay", [0, 2])
def test_duplicate_start_preserves_output_prefix(self, tmp_path, delay):
wrapper, proton = make_proton_wrapper(
tmp_path, proton_graph_attribution=True, delay_iterations=delay
)
with wrapper.capture_cuda_graphs():
pass
wrapper.set_output_name("first")
wrapper.start()
wrapper.set_output_name("duplicate")
wrapper.start()
for _ in range(delay):
wrapper.step()
wrapper.stop()
assert len(list(tmp_path.glob("proton_first_*.hatchet"))) == 1
assert not list(tmp_path.glob("proton_duplicate_*"))
wrapper.set_output_name("second")
wrapper.start()
for _ in range(delay):
wrapper.step()
wrapper.stop()
assert len(list(tmp_path.glob("proton_second_*.hatchet"))) == 1
def test_failed_export_clears_activity_and_allows_next_interval(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
proton.data.get.side_effect = [OSError("disk full"), []]
wrapper.start()
wrapper.stop()
proton.data.clear.assert_called_with(7, 1)
wrapper.start()
wrapper.stop()
proton.data.clear.assert_called_with(7, 2)
assert len(list(tmp_path.glob("*_run1.hatchet"))) == 1
def test_cuda_graph_context_deactivates_after_capture_error(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with (
pytest.raises(RuntimeError, match="capture failed"),
wrapper.capture_cuda_graphs(),
):
raise RuntimeError("capture failed")
proton.deactivate.assert_called_once_with(session=7, flushing=True)
def test_cuda_graph_capture_deactivates_when_phase_advance_fails(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
with (
pytest.raises(RuntimeError, match="advance failed"),
wrapper.capture_cuda_graphs(),
):
pass
proton.deactivate.assert_called_once_with(session=7, flushing=True)
proton.data.clear.assert_not_called()
def test_cuda_graph_stop_deactivates_when_phase_advance_fails(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
proton.data.clear.reset_mock()
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
wrapper._start()
with pytest.raises(RuntimeError, match="advance failed"):
wrapper._stop()
proton.deactivate.assert_called_with(session=7, flushing=True)
proton.data.clear.assert_not_called()
def test_shutdown_finalizes_cuda_graph_capture_session(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
wrapper.shutdown()
wrapper.shutdown()
proton.finalize.assert_called_once_with(session=7)
def test_missing_proton_has_actionable_error(self, tmp_path):
config = ProfilerConfig(profiler="proton", proton_profiler_dir=str(tmp_path))
with (
@@ -638,6 +871,7 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
),
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
):
wrapper.return_value.has_cuda_graph_session = False
Worker.profile(worker, profile_prefix="first")
Worker.profile(worker, is_start=False)
Worker.profile(worker, profile_prefix="second")
@@ -647,3 +881,157 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
call(worker.profiler_config, worker_name="second_rank1"),
]
assert wrapper.return_value.start.call_count == 2
@_requires_cuda_for_proton
def test_gpu_worker_reuses_cuda_graph_proton_session():
worker = MagicMock()
worker.rank = 1
worker.profiler = MagicMock(spec=ProtonProfilerWrapper)
worker.profiler.has_cuda_graph_session = True
worker.profiler_config.profiler = "proton"
with patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank1",
):
Worker.profile(worker, profile_prefix="first")
Worker.profile(worker, is_start=False)
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
worker.profiler.start.assert_called_once_with()
worker.profiler.stop.assert_called_once_with()
@_requires_cuda_for_proton
@pytest.mark.parametrize("runner", ["attribution_off", "v1", "no_capture"])
def test_proton_not_initialized_without_capture(runner):
worker = MagicMock()
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config.proton_graph_attribution = runner != "attribution_off"
worker.use_v2_model_runner = runner != "v1"
worker.model_runner.needs_cudagraph_capture.return_value = runner != "no_capture"
context = Worker._get_cudagraph_capture_context(worker)
assert worker.profiler is None
with context:
pass
@_requires_cuda_for_proton
def test_proton_initializes_before_cuda_graph_capture():
class FakeProtonProfiler:
def __init__(self, config, worker_name):
self.config = config
self.worker_name = worker_name
self.capture_context = nullcontext()
def capture_cuda_graphs(self):
return self.capture_context
worker = MagicMock()
worker.rank = 2
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config.proton_graph_attribution = True
worker.use_v2_model_runner = True
worker.model_runner.needs_cudagraph_capture.return_value = True
with (
patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank2",
),
patch(
"vllm.profiler.wrapper.ProtonProfilerWrapper",
FakeProtonProfiler,
),
):
context = Worker._get_cudagraph_capture_context(worker)
assert worker.profiler.config is worker.profiler_config
assert worker.profiler.worker_name == "rank2"
assert context is worker.profiler.capture_context
@_requires_cuda_for_proton
@pytest.mark.parametrize("context", ["shadow", "python"])
@pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"])
def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format):
"""Both intervals contain replay kernels, without capture-only activity."""
import json
import torch
import triton
import triton.profiler as proton
from packaging.version import Version
if Version(triton.__version__) < Version("3.7"):
pytest.skip("Graph attribution requires Triton >= 3.7")
wrapper = ProtonProfilerWrapper(
ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
proton_context=context,
proton_output_format=output_format,
),
worker_name="gpu",
)
worker = SimpleNamespace(
profiler=wrapper,
use_v2_model_runner=True,
)
x = torch.ones(1024, device="cuda")
graph = torch.cuda.CUDAGraph()
def capture_only():
x.add_(1)
def captured_add():
x.add_(1)
def kernel_metrics(value):
if isinstance(value, list):
return [metric for child in value for metric in kernel_metrics(child)]
if isinstance(value, dict):
metrics = value.get("metrics", {})
return ([metrics] if metrics.get("count", 0) else []) + [
metric for child in value.values() for metric in kernel_metrics(child)
]
return []
try:
with Worker._get_cudagraph_capture_context(worker):
with proton.scope("capture_only"):
capture_only()
torch.accelerator.synchronize()
with torch.cuda.graph(graph), proton.scope("captured_add"):
captured_add()
for run in range(2):
wrapper.start()
with wrapper.annotate_context_manager(f"replay_{run}"):
graph.replay()
wrapper.stop()
(path,) = tmp_path.glob(f"*_run{run}.{output_format}")
if output_format == "hatchet_msgpack":
import msgpack
data = msgpack.unpackb(path.read_bytes())
else:
data = json.loads(path.read_text())
serialized = json.dumps(data)
assert "<captured_at>" in serialized
assert "captured_add" in serialized
assert "capture_only" not in serialized
assert f"replay_{1 - run}" not in serialized
metrics = kernel_metrics(data)
assert sum(metric["count"] for metric in metrics) == 1
assert sum(metric["time (ns)"] for metric in metrics) > 0
torch.testing.assert_close(x, torch.full_like(x, 4))
finally:
wrapper.shutdown()
assert not list(tmp_path.glob(".proton_cuda_graph_session*"))