[Profiler] Add Proton CUDA graph attribution for MRV2 (#51084)
Signed-off-by: Luossu <zhluosuu@outlook.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -350,17 +350,24 @@ def test_profiler_entered_during_capture():
|
||||
|
||||
|
||||
def make_proton(session_id: int | None = 7):
|
||||
data = SimpleNamespace(
|
||||
advance_phase=Mock(side_effect=range(1, 100)),
|
||||
clear=Mock(),
|
||||
get=Mock(return_value={"traceEvents": []}),
|
||||
get_msgpack=Mock(return_value=b"profile"),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
start=Mock(return_value=session_id),
|
||||
activate=Mock(),
|
||||
deactivate=Mock(),
|
||||
finalize=Mock(),
|
||||
scope=Mock(return_value=nullcontext()),
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
def make_proton_wrapper(
|
||||
tmp_path, proton=None, triton_version="3.6.0", **config_overrides
|
||||
tmp_path, proton=None, triton_version="3.7.0", **config_overrides
|
||||
):
|
||||
proton = proton or make_proton()
|
||||
config = ProfilerConfig(
|
||||
@@ -415,6 +422,13 @@ class TestProtonConfig:
|
||||
},
|
||||
"requires proton_data",
|
||||
),
|
||||
(
|
||||
{
|
||||
"proton_data": "trace",
|
||||
"proton_graph_attribution": True,
|
||||
},
|
||||
"requires proton_data='tree'",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_option_combinations(self, tmp_path, options, message):
|
||||
@@ -450,6 +464,7 @@ class TestProtonConfig:
|
||||
("proton_mode", "pcsampling"),
|
||||
("proton_hook", "triton"),
|
||||
("proton_output_format", "chrome_trace"),
|
||||
("proton_graph_attribution", True),
|
||||
],
|
||||
)
|
||||
def test_rejects_proton_options_for_other_profilers(self, field, value):
|
||||
@@ -478,15 +493,101 @@ class TestProtonConfig:
|
||||
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.NONE),
|
||||
)
|
||||
|
||||
def test_rejects_proton_when_cuda_graphs_are_enabled(self, tmp_path):
|
||||
with pytest.raises(ValueError, match="requires CUDA graphs to be disabled"):
|
||||
@pytest.mark.parametrize("encoder_only", [False, True])
|
||||
@pytest.mark.parametrize("attribution", [False, True])
|
||||
def test_cuda_graphs_require_attribution(self, tmp_path, encoder_only, attribution):
|
||||
# Encoder graphs are independent of the decoder's cudagraph_mode.
|
||||
expected = (
|
||||
nullcontext()
|
||||
if attribution
|
||||
else pytest.raises(
|
||||
ValueError, match="requires proton_graph_attribution=True"
|
||||
)
|
||||
)
|
||||
with expected:
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=attribution,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=(
|
||||
CUDAGraphMode.NONE if encoder_only else CUDAGraphMode.FULL
|
||||
),
|
||||
cudagraph_mm_encoder=encoder_only,
|
||||
),
|
||||
)
|
||||
|
||||
def test_validates_default_cuda_graph_mode_after_resolution(self, tmp_path):
|
||||
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton", proton_profiler_dir=str(tmp_path)
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
[
|
||||
"periodic_flushing",
|
||||
"periodic_flushing:format=hatchet",
|
||||
"PERIODIC_FLUSHING:format=hatchet",
|
||||
],
|
||||
)
|
||||
def test_graph_attribution_rejects_periodic_flushing(self, tmp_path, mode):
|
||||
# Reject before Proton's native phase manager can abort the worker.
|
||||
with pytest.raises(ValueError, match="incompatible with periodic_flushing"):
|
||||
ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
proton_mode=mode,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("attribution", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"mode", ["pcsampling", "pcsampling:interval=100", "PcSampling:interval=100"]
|
||||
)
|
||||
@pytest.mark.parametrize("encoder_only", [False, True])
|
||||
def test_pcsampling_requires_graphs_disabled(
|
||||
self, tmp_path, attribution, mode, encoder_only
|
||||
):
|
||||
with pytest.raises(ValueError, match="PC sampling requires CUDA graphs"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=attribution,
|
||||
proton_mode=mode,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.NONE
|
||||
if encoder_only
|
||||
else CUDAGraphMode.FULL,
|
||||
cudagraph_mm_encoder=encoder_only,
|
||||
),
|
||||
)
|
||||
|
||||
def test_ordinary_proton_keeps_mrv1_graph_restriction(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton", proton_profiler_dir=str(tmp_path)
|
||||
),
|
||||
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL),
|
||||
)
|
||||
|
||||
def test_graph_attribution_rejects_mrv1(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
with pytest.raises(ValueError, match="requires the V2 model runner"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -564,7 +665,20 @@ class TestProtonProfilerWrapper:
|
||||
)
|
||||
def test_newer_features_reject_triton_3_6(self, tmp_path, option, value, feature):
|
||||
with pytest.raises(RuntimeError, match=feature):
|
||||
make_proton_wrapper(tmp_path, **{option: value})
|
||||
make_proton_wrapper(tmp_path, triton_version="3.6.0", **{option: value})
|
||||
|
||||
@pytest.mark.parametrize("version", ["3.6.0", "unknown"])
|
||||
def test_graph_attribution_requires_phase_api(self, tmp_path, version):
|
||||
with pytest.raises(RuntimeError, match="requires Triton >= 3.7"):
|
||||
make_proton_wrapper(
|
||||
tmp_path, triton_version=version, proton_graph_attribution=True
|
||||
)
|
||||
|
||||
def test_ordinary_profiling_still_supports_triton_3_6(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, triton_version="3.6.0")
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.finalize.assert_called_once_with(session=7)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("option", "value"),
|
||||
@@ -583,6 +697,125 @@ class TestProtonProfilerWrapper:
|
||||
with pytest.raises(RuntimeError, match="does not support selecting"):
|
||||
make_proton_wrapper(tmp_path, proton, proton_output_format="hatchet")
|
||||
|
||||
def test_cuda_graph_tree_phase_is_written_at_stop(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(
|
||||
tmp_path,
|
||||
proton_context="python",
|
||||
proton_graph_attribution=True,
|
||||
)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
proton.start.assert_called_once()
|
||||
proton.deactivate.assert_not_called()
|
||||
|
||||
capture_args = proton.start.call_args.kwargs
|
||||
assert capture_args["context"] == "python"
|
||||
assert capture_args["data"] == "tree"
|
||||
proton.data.advance_phase.assert_called_once_with(7)
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_called_once_with(7, 0)
|
||||
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
assert proton.data.get.call_args_list == [call(7, 1), call(7, 2)]
|
||||
proton.data.clear.assert_has_calls([call(7, 0), call(7, 1), call(7, 2)])
|
||||
output_names = sorted(tmp_path.glob("proton_rank_3_*.hatchet"))
|
||||
assert len(output_names) == 2
|
||||
assert output_names[0].name.endswith("_run0.hatchet")
|
||||
assert output_names[1].name.endswith("_run1.hatchet")
|
||||
|
||||
def test_capture_is_noop_without_opt_in(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.start.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("delay", [0, 2])
|
||||
def test_duplicate_start_preserves_output_prefix(self, tmp_path, delay):
|
||||
wrapper, proton = make_proton_wrapper(
|
||||
tmp_path, proton_graph_attribution=True, delay_iterations=delay
|
||||
)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
wrapper.set_output_name("first")
|
||||
wrapper.start()
|
||||
wrapper.set_output_name("duplicate")
|
||||
wrapper.start()
|
||||
for _ in range(delay):
|
||||
wrapper.step()
|
||||
wrapper.stop()
|
||||
assert len(list(tmp_path.glob("proton_first_*.hatchet"))) == 1
|
||||
assert not list(tmp_path.glob("proton_duplicate_*"))
|
||||
wrapper.set_output_name("second")
|
||||
wrapper.start()
|
||||
for _ in range(delay):
|
||||
wrapper.step()
|
||||
wrapper.stop()
|
||||
assert len(list(tmp_path.glob("proton_second_*.hatchet"))) == 1
|
||||
|
||||
def test_failed_export_clears_activity_and_allows_next_interval(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.data.get.side_effect = [OSError("disk full"), []]
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.data.clear.assert_called_with(7, 1)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.data.clear.assert_called_with(7, 2)
|
||||
assert len(list(tmp_path.glob("*_run1.hatchet"))) == 1
|
||||
|
||||
def test_cuda_graph_context_deactivates_after_capture_error(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="capture failed"),
|
||||
wrapper.capture_cuda_graphs(),
|
||||
):
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
|
||||
def test_cuda_graph_capture_deactivates_when_phase_advance_fails(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="advance failed"),
|
||||
wrapper.capture_cuda_graphs(),
|
||||
):
|
||||
pass
|
||||
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_not_called()
|
||||
|
||||
def test_cuda_graph_stop_deactivates_when_phase_advance_fails(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.data.clear.reset_mock()
|
||||
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
|
||||
|
||||
wrapper._start()
|
||||
with pytest.raises(RuntimeError, match="advance failed"):
|
||||
wrapper._stop()
|
||||
|
||||
proton.deactivate.assert_called_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_not_called()
|
||||
|
||||
def test_shutdown_finalizes_cuda_graph_capture_session(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
|
||||
wrapper.shutdown()
|
||||
wrapper.shutdown()
|
||||
|
||||
proton.finalize.assert_called_once_with(session=7)
|
||||
|
||||
def test_missing_proton_has_actionable_error(self, tmp_path):
|
||||
config = ProfilerConfig(profiler="proton", proton_profiler_dir=str(tmp_path))
|
||||
with (
|
||||
@@ -638,6 +871,7 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
wrapper.return_value.has_cuda_graph_session = False
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
Worker.profile(worker, is_start=False)
|
||||
Worker.profile(worker, profile_prefix="second")
|
||||
@@ -647,3 +881,157 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
call(worker.profiler_config, worker_name="second_rank1"),
|
||||
]
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_reuses_cuda_graph_proton_session():
|
||||
worker = MagicMock()
|
||||
worker.rank = 1
|
||||
worker.profiler = MagicMock(spec=ProtonProfilerWrapper)
|
||||
worker.profiler.has_cuda_graph_session = True
|
||||
worker.profiler_config.profiler = "proton"
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
):
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
Worker.profile(worker, is_start=False)
|
||||
|
||||
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@pytest.mark.parametrize("runner", ["attribution_off", "v1", "no_capture"])
|
||||
def test_proton_not_initialized_without_capture(runner):
|
||||
worker = MagicMock()
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config.proton_graph_attribution = runner != "attribution_off"
|
||||
worker.use_v2_model_runner = runner != "v1"
|
||||
worker.model_runner.needs_cudagraph_capture.return_value = runner != "no_capture"
|
||||
|
||||
context = Worker._get_cudagraph_capture_context(worker)
|
||||
|
||||
assert worker.profiler is None
|
||||
with context:
|
||||
pass
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_proton_initializes_before_cuda_graph_capture():
|
||||
class FakeProtonProfiler:
|
||||
def __init__(self, config, worker_name):
|
||||
self.config = config
|
||||
self.worker_name = worker_name
|
||||
self.capture_context = nullcontext()
|
||||
|
||||
def capture_cuda_graphs(self):
|
||||
return self.capture_context
|
||||
|
||||
worker = MagicMock()
|
||||
worker.rank = 2
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config.proton_graph_attribution = True
|
||||
worker.use_v2_model_runner = True
|
||||
worker.model_runner.needs_cudagraph_capture.return_value = True
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank2",
|
||||
),
|
||||
patch(
|
||||
"vllm.profiler.wrapper.ProtonProfilerWrapper",
|
||||
FakeProtonProfiler,
|
||||
),
|
||||
):
|
||||
context = Worker._get_cudagraph_capture_context(worker)
|
||||
|
||||
assert worker.profiler.config is worker.profiler_config
|
||||
assert worker.profiler.worker_name == "rank2"
|
||||
assert context is worker.profiler.capture_context
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@pytest.mark.parametrize("context", ["shadow", "python"])
|
||||
@pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"])
|
||||
def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format):
|
||||
"""Both intervals contain replay kernels, without capture-only activity."""
|
||||
import json
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.profiler as proton
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(triton.__version__) < Version("3.7"):
|
||||
pytest.skip("Graph attribution requires Triton >= 3.7")
|
||||
|
||||
wrapper = ProtonProfilerWrapper(
|
||||
ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
proton_context=context,
|
||||
proton_output_format=output_format,
|
||||
),
|
||||
worker_name="gpu",
|
||||
)
|
||||
worker = SimpleNamespace(
|
||||
profiler=wrapper,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
x = torch.ones(1024, device="cuda")
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
|
||||
def capture_only():
|
||||
x.add_(1)
|
||||
|
||||
def captured_add():
|
||||
x.add_(1)
|
||||
|
||||
def kernel_metrics(value):
|
||||
if isinstance(value, list):
|
||||
return [metric for child in value for metric in kernel_metrics(child)]
|
||||
if isinstance(value, dict):
|
||||
metrics = value.get("metrics", {})
|
||||
return ([metrics] if metrics.get("count", 0) else []) + [
|
||||
metric for child in value.values() for metric in kernel_metrics(child)
|
||||
]
|
||||
return []
|
||||
|
||||
try:
|
||||
with Worker._get_cudagraph_capture_context(worker):
|
||||
with proton.scope("capture_only"):
|
||||
capture_only()
|
||||
torch.accelerator.synchronize()
|
||||
with torch.cuda.graph(graph), proton.scope("captured_add"):
|
||||
captured_add()
|
||||
for run in range(2):
|
||||
wrapper.start()
|
||||
with wrapper.annotate_context_manager(f"replay_{run}"):
|
||||
graph.replay()
|
||||
wrapper.stop()
|
||||
(path,) = tmp_path.glob(f"*_run{run}.{output_format}")
|
||||
if output_format == "hatchet_msgpack":
|
||||
import msgpack
|
||||
|
||||
data = msgpack.unpackb(path.read_bytes())
|
||||
else:
|
||||
data = json.loads(path.read_text())
|
||||
serialized = json.dumps(data)
|
||||
assert "<captured_at>" in serialized
|
||||
assert "captured_add" in serialized
|
||||
assert "capture_only" not in serialized
|
||||
assert f"replay_{1 - run}" not in serialized
|
||||
metrics = kernel_metrics(data)
|
||||
assert sum(metric["count"] for metric in metrics) == 1
|
||||
assert sum(metric["time (ns)"] for metric in metrics) > 0
|
||||
torch.testing.assert_close(x, torch.full_like(x, 4))
|
||||
finally:
|
||||
wrapper.shutdown()
|
||||
assert not list(tmp_path.glob(".proton_cuda_graph_session*"))
|
||||
|
||||
Reference in New Issue
Block a user