[Profiler] Add Proton CUDA graph attribution for MRV2 (#51084)
Signed-off-by: Luossu <zhluosuu@outlook.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -90,18 +90,18 @@ $ curl -X POST http://localhost:8000/stop_profile
|
||||
is Triton's GPU profiler. It can collect a low-overhead aggregate tree or a
|
||||
Chrome trace and works through the same vLLM profiling controls as the PyTorch
|
||||
and CUDA profilers. Proton currently supports NVIDIA GPUs through CUPTI and
|
||||
requires eager execution.
|
||||
supports CUDA graph attribution.
|
||||
|
||||
Start a server with a local output directory:
|
||||
Start a server with a local output directory and graph attribution:
|
||||
|
||||
```bash
|
||||
vllm serve meta-llama/Llama-3.1-8B-Instruct \
|
||||
--enforce-eager \
|
||||
--profiler-config '{
|
||||
"profiler": "proton",
|
||||
"proton_profiler_dir": "./proton_profile",
|
||||
"proton_output_format": "hatchet",
|
||||
"proton_hook": "triton"
|
||||
"proton_hook": "triton",
|
||||
"proton_graph_attribution": true
|
||||
}'
|
||||
```
|
||||
|
||||
@@ -111,7 +111,7 @@ rank-qualified output name, such as
|
||||
`proton_dp0_pp0_tp0_dcp0_ep0_rank0_pid1234_0123456789abcdef0123456789abcdef_run0.hatchet`,
|
||||
so distributed workers, restarted servers, and repeated profiling runs do not
|
||||
overwrite one another. A `profile_prefix` is included when supplied. Each
|
||||
profile is finalized by `/stop_profile` and is ready to inspect immediately.
|
||||
profile is written by `/stop_profile` and is ready to inspect immediately.
|
||||
|
||||
The Proton-specific options are:
|
||||
|
||||
@@ -121,21 +121,39 @@ The Proton-specific options are:
|
||||
- `proton_mode`: an optional backend mode string
|
||||
- `proton_hook`: `triton` to record Triton launch metadata, or unset
|
||||
- `proton_output_format`: `hatchet`, `hatchet_msgpack`, `chrome_trace`, or unset
|
||||
- `proton_graph_attribution`: observe CUDA graph capture for replay attribution;
|
||||
disabled by default and requires `proton_data: "tree"`
|
||||
|
||||
`hatchet` and `hatchet_msgpack` require `proton_data: "tree"`, while
|
||||
`chrome_trace` requires `proton_data: "trace"`. CUDA graph profiling is not yet
|
||||
supported, so `--enforce-eager` is required for every Proton mode.
|
||||
`chrome_trace` requires `proton_data: "trace"`.
|
||||
|
||||
Automatic backend selection is recommended. vLLM currently supports Proton's
|
||||
`cupti` backend on NVIDIA GPUs. ROCm support is not yet available. vLLM does not
|
||||
expose Proton's experimental instrumentation backend because current upstream
|
||||
Triton builds can produce profiles without timing metrics. Backend-specific
|
||||
modes, including `pcsampling`, can be selected with `proton_mode`.
|
||||
can produce profiles without timing metrics. When `proton_graph_attribution` is
|
||||
enabled, Proton observes vLLM's CUDA graph capture with the configured profiling
|
||||
session active, then deactivates that same session until profiling starts. This
|
||||
lets later profiles attribute replayed kernels without retaining model-startup
|
||||
activity. Backend-specific modes can be selected with `proton_mode`;
|
||||
`pcsampling` synchronizes the CUDA context and therefore requires
|
||||
`--enforce-eager`. When CUDA graphs are enabled (including encoder graphs),
|
||||
Proton requires `proton_graph_attribution: true` to collect replayed kernels.
|
||||
For Chrome traces, disable CUDA graphs with `--enforce-eager`.
|
||||
|
||||
Triton 3.6 supports explicit `hatchet` and `chrome_trace` output. The
|
||||
`hatchet_msgpack` format and `periodic_flushing` mode require Triton 3.7 or
|
||||
newer; vLLM rejects these options with the detected version before starting a
|
||||
session.
|
||||
CUDA graph-attributed profiles support repeated `start_profile`/`stop_profile`
|
||||
runs. Each stop flushes and writes one tree-data phase while preserving the
|
||||
graph-aware session. Without `proton_graph_attribution`, each `stop_profile`
|
||||
instead finalizes and writes an independent Proton session.
|
||||
|
||||
CUDA graph attribution requires Triton 3.7 or newer. It uses the phase data API
|
||||
to discard graph-capture activity and separate profiling runs. The `hatchet_msgpack`
|
||||
output format and `periodic_flushing` mode also require Triton 3.7 or newer.
|
||||
`periodic_flushing` cannot be combined with graph attribution because both
|
||||
manage the session's data phases. Ordinary Proton profiling remains available
|
||||
with Triton 3.6.
|
||||
|
||||
Graph attribution retains capture metadata for the worker lifetime. With eager
|
||||
execution or no graphs to capture, profiling uses ordinary independent sessions.
|
||||
|
||||
Inspect tree profiles with:
|
||||
|
||||
|
||||
@@ -350,17 +350,24 @@ def test_profiler_entered_during_capture():
|
||||
|
||||
|
||||
def make_proton(session_id: int | None = 7):
|
||||
data = SimpleNamespace(
|
||||
advance_phase=Mock(side_effect=range(1, 100)),
|
||||
clear=Mock(),
|
||||
get=Mock(return_value={"traceEvents": []}),
|
||||
get_msgpack=Mock(return_value=b"profile"),
|
||||
)
|
||||
return SimpleNamespace(
|
||||
start=Mock(return_value=session_id),
|
||||
activate=Mock(),
|
||||
deactivate=Mock(),
|
||||
finalize=Mock(),
|
||||
scope=Mock(return_value=nullcontext()),
|
||||
data=data,
|
||||
)
|
||||
|
||||
|
||||
def make_proton_wrapper(
|
||||
tmp_path, proton=None, triton_version="3.6.0", **config_overrides
|
||||
tmp_path, proton=None, triton_version="3.7.0", **config_overrides
|
||||
):
|
||||
proton = proton or make_proton()
|
||||
config = ProfilerConfig(
|
||||
@@ -415,6 +422,13 @@ class TestProtonConfig:
|
||||
},
|
||||
"requires proton_data",
|
||||
),
|
||||
(
|
||||
{
|
||||
"proton_data": "trace",
|
||||
"proton_graph_attribution": True,
|
||||
},
|
||||
"requires proton_data='tree'",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_option_combinations(self, tmp_path, options, message):
|
||||
@@ -450,6 +464,7 @@ class TestProtonConfig:
|
||||
("proton_mode", "pcsampling"),
|
||||
("proton_hook", "triton"),
|
||||
("proton_output_format", "chrome_trace"),
|
||||
("proton_graph_attribution", True),
|
||||
],
|
||||
)
|
||||
def test_rejects_proton_options_for_other_profilers(self, field, value):
|
||||
@@ -478,15 +493,101 @@ class TestProtonConfig:
|
||||
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.NONE),
|
||||
)
|
||||
|
||||
def test_rejects_proton_when_cuda_graphs_are_enabled(self, tmp_path):
|
||||
with pytest.raises(ValueError, match="requires CUDA graphs to be disabled"):
|
||||
@pytest.mark.parametrize("encoder_only", [False, True])
|
||||
@pytest.mark.parametrize("attribution", [False, True])
|
||||
def test_cuda_graphs_require_attribution(self, tmp_path, encoder_only, attribution):
|
||||
# Encoder graphs are independent of the decoder's cudagraph_mode.
|
||||
expected = (
|
||||
nullcontext()
|
||||
if attribution
|
||||
else pytest.raises(
|
||||
ValueError, match="requires proton_graph_attribution=True"
|
||||
)
|
||||
)
|
||||
with expected:
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=attribution,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=(
|
||||
CUDAGraphMode.NONE if encoder_only else CUDAGraphMode.FULL
|
||||
),
|
||||
cudagraph_mm_encoder=encoder_only,
|
||||
),
|
||||
)
|
||||
|
||||
def test_validates_default_cuda_graph_mode_after_resolution(self, tmp_path):
|
||||
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton", proton_profiler_dir=str(tmp_path)
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
[
|
||||
"periodic_flushing",
|
||||
"periodic_flushing:format=hatchet",
|
||||
"PERIODIC_FLUSHING:format=hatchet",
|
||||
],
|
||||
)
|
||||
def test_graph_attribution_rejects_periodic_flushing(self, tmp_path, mode):
|
||||
# Reject before Proton's native phase manager can abort the worker.
|
||||
with pytest.raises(ValueError, match="incompatible with periodic_flushing"):
|
||||
ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
proton_mode=mode,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("attribution", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"mode", ["pcsampling", "pcsampling:interval=100", "PcSampling:interval=100"]
|
||||
)
|
||||
@pytest.mark.parametrize("encoder_only", [False, True])
|
||||
def test_pcsampling_requires_graphs_disabled(
|
||||
self, tmp_path, attribution, mode, encoder_only
|
||||
):
|
||||
with pytest.raises(ValueError, match="PC sampling requires CUDA graphs"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=attribution,
|
||||
proton_mode=mode,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.NONE
|
||||
if encoder_only
|
||||
else CUDAGraphMode.FULL,
|
||||
cudagraph_mm_encoder=encoder_only,
|
||||
),
|
||||
)
|
||||
|
||||
def test_ordinary_proton_keeps_mrv1_graph_restriction(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton", proton_profiler_dir=str(tmp_path)
|
||||
),
|
||||
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL),
|
||||
)
|
||||
|
||||
def test_graph_attribution_rejects_mrv1(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
with pytest.raises(ValueError, match="requires the V2 model runner"):
|
||||
VllmConfig(
|
||||
profiler_config=ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -564,7 +665,20 @@ class TestProtonProfilerWrapper:
|
||||
)
|
||||
def test_newer_features_reject_triton_3_6(self, tmp_path, option, value, feature):
|
||||
with pytest.raises(RuntimeError, match=feature):
|
||||
make_proton_wrapper(tmp_path, **{option: value})
|
||||
make_proton_wrapper(tmp_path, triton_version="3.6.0", **{option: value})
|
||||
|
||||
@pytest.mark.parametrize("version", ["3.6.0", "unknown"])
|
||||
def test_graph_attribution_requires_phase_api(self, tmp_path, version):
|
||||
with pytest.raises(RuntimeError, match="requires Triton >= 3.7"):
|
||||
make_proton_wrapper(
|
||||
tmp_path, triton_version=version, proton_graph_attribution=True
|
||||
)
|
||||
|
||||
def test_ordinary_profiling_still_supports_triton_3_6(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, triton_version="3.6.0")
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.finalize.assert_called_once_with(session=7)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("option", "value"),
|
||||
@@ -583,6 +697,125 @@ class TestProtonProfilerWrapper:
|
||||
with pytest.raises(RuntimeError, match="does not support selecting"):
|
||||
make_proton_wrapper(tmp_path, proton, proton_output_format="hatchet")
|
||||
|
||||
def test_cuda_graph_tree_phase_is_written_at_stop(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(
|
||||
tmp_path,
|
||||
proton_context="python",
|
||||
proton_graph_attribution=True,
|
||||
)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
proton.start.assert_called_once()
|
||||
proton.deactivate.assert_not_called()
|
||||
|
||||
capture_args = proton.start.call_args.kwargs
|
||||
assert capture_args["context"] == "python"
|
||||
assert capture_args["data"] == "tree"
|
||||
proton.data.advance_phase.assert_called_once_with(7)
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_called_once_with(7, 0)
|
||||
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
assert proton.data.get.call_args_list == [call(7, 1), call(7, 2)]
|
||||
proton.data.clear.assert_has_calls([call(7, 0), call(7, 1), call(7, 2)])
|
||||
output_names = sorted(tmp_path.glob("proton_rank_3_*.hatchet"))
|
||||
assert len(output_names) == 2
|
||||
assert output_names[0].name.endswith("_run0.hatchet")
|
||||
assert output_names[1].name.endswith("_run1.hatchet")
|
||||
|
||||
def test_capture_is_noop_without_opt_in(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.start.assert_not_called()
|
||||
|
||||
@pytest.mark.parametrize("delay", [0, 2])
|
||||
def test_duplicate_start_preserves_output_prefix(self, tmp_path, delay):
|
||||
wrapper, proton = make_proton_wrapper(
|
||||
tmp_path, proton_graph_attribution=True, delay_iterations=delay
|
||||
)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
wrapper.set_output_name("first")
|
||||
wrapper.start()
|
||||
wrapper.set_output_name("duplicate")
|
||||
wrapper.start()
|
||||
for _ in range(delay):
|
||||
wrapper.step()
|
||||
wrapper.stop()
|
||||
assert len(list(tmp_path.glob("proton_first_*.hatchet"))) == 1
|
||||
assert not list(tmp_path.glob("proton_duplicate_*"))
|
||||
wrapper.set_output_name("second")
|
||||
wrapper.start()
|
||||
for _ in range(delay):
|
||||
wrapper.step()
|
||||
wrapper.stop()
|
||||
assert len(list(tmp_path.glob("proton_second_*.hatchet"))) == 1
|
||||
|
||||
def test_failed_export_clears_activity_and_allows_next_interval(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.data.get.side_effect = [OSError("disk full"), []]
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.data.clear.assert_called_with(7, 1)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
proton.data.clear.assert_called_with(7, 2)
|
||||
assert len(list(tmp_path.glob("*_run1.hatchet"))) == 1
|
||||
|
||||
def test_cuda_graph_context_deactivates_after_capture_error(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="capture failed"),
|
||||
wrapper.capture_cuda_graphs(),
|
||||
):
|
||||
raise RuntimeError("capture failed")
|
||||
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
|
||||
def test_cuda_graph_capture_deactivates_when_phase_advance_fails(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
|
||||
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="advance failed"),
|
||||
wrapper.capture_cuda_graphs(),
|
||||
):
|
||||
pass
|
||||
|
||||
proton.deactivate.assert_called_once_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_not_called()
|
||||
|
||||
def test_cuda_graph_stop_deactivates_when_phase_advance_fails(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
proton.data.clear.reset_mock()
|
||||
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
|
||||
|
||||
wrapper._start()
|
||||
with pytest.raises(RuntimeError, match="advance failed"):
|
||||
wrapper._stop()
|
||||
|
||||
proton.deactivate.assert_called_with(session=7, flushing=True)
|
||||
proton.data.clear.assert_not_called()
|
||||
|
||||
def test_shutdown_finalizes_cuda_graph_capture_session(self, tmp_path):
|
||||
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
|
||||
with wrapper.capture_cuda_graphs():
|
||||
pass
|
||||
|
||||
wrapper.shutdown()
|
||||
wrapper.shutdown()
|
||||
|
||||
proton.finalize.assert_called_once_with(session=7)
|
||||
|
||||
def test_missing_proton_has_actionable_error(self, tmp_path):
|
||||
config = ProfilerConfig(profiler="proton", proton_profiler_dir=str(tmp_path))
|
||||
with (
|
||||
@@ -638,6 +871,7 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
wrapper.return_value.has_cuda_graph_session = False
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
Worker.profile(worker, is_start=False)
|
||||
Worker.profile(worker, profile_prefix="second")
|
||||
@@ -647,3 +881,157 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
call(worker.profiler_config, worker_name="second_rank1"),
|
||||
]
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_reuses_cuda_graph_proton_session():
|
||||
worker = MagicMock()
|
||||
worker.rank = 1
|
||||
worker.profiler = MagicMock(spec=ProtonProfilerWrapper)
|
||||
worker.profiler.has_cuda_graph_session = True
|
||||
worker.profiler_config.profiler = "proton"
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
):
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
Worker.profile(worker, is_start=False)
|
||||
|
||||
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@pytest.mark.parametrize("runner", ["attribution_off", "v1", "no_capture"])
|
||||
def test_proton_not_initialized_without_capture(runner):
|
||||
worker = MagicMock()
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config.proton_graph_attribution = runner != "attribution_off"
|
||||
worker.use_v2_model_runner = runner != "v1"
|
||||
worker.model_runner.needs_cudagraph_capture.return_value = runner != "no_capture"
|
||||
|
||||
context = Worker._get_cudagraph_capture_context(worker)
|
||||
|
||||
assert worker.profiler is None
|
||||
with context:
|
||||
pass
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_proton_initializes_before_cuda_graph_capture():
|
||||
class FakeProtonProfiler:
|
||||
def __init__(self, config, worker_name):
|
||||
self.config = config
|
||||
self.worker_name = worker_name
|
||||
self.capture_context = nullcontext()
|
||||
|
||||
def capture_cuda_graphs(self):
|
||||
return self.capture_context
|
||||
|
||||
worker = MagicMock()
|
||||
worker.rank = 2
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config.proton_graph_attribution = True
|
||||
worker.use_v2_model_runner = True
|
||||
worker.model_runner.needs_cudagraph_capture.return_value = True
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank2",
|
||||
),
|
||||
patch(
|
||||
"vllm.profiler.wrapper.ProtonProfilerWrapper",
|
||||
FakeProtonProfiler,
|
||||
),
|
||||
):
|
||||
context = Worker._get_cudagraph_capture_context(worker)
|
||||
|
||||
assert worker.profiler.config is worker.profiler_config
|
||||
assert worker.profiler.worker_name == "rank2"
|
||||
assert context is worker.profiler.capture_context
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@pytest.mark.parametrize("context", ["shadow", "python"])
|
||||
@pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"])
|
||||
def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format):
|
||||
"""Both intervals contain replay kernels, without capture-only activity."""
|
||||
import json
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.profiler as proton
|
||||
from packaging.version import Version
|
||||
|
||||
if Version(triton.__version__) < Version("3.7"):
|
||||
pytest.skip("Graph attribution requires Triton >= 3.7")
|
||||
|
||||
wrapper = ProtonProfilerWrapper(
|
||||
ProfilerConfig(
|
||||
profiler="proton",
|
||||
proton_profiler_dir=str(tmp_path),
|
||||
proton_graph_attribution=True,
|
||||
proton_context=context,
|
||||
proton_output_format=output_format,
|
||||
),
|
||||
worker_name="gpu",
|
||||
)
|
||||
worker = SimpleNamespace(
|
||||
profiler=wrapper,
|
||||
use_v2_model_runner=True,
|
||||
)
|
||||
x = torch.ones(1024, device="cuda")
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
|
||||
def capture_only():
|
||||
x.add_(1)
|
||||
|
||||
def captured_add():
|
||||
x.add_(1)
|
||||
|
||||
def kernel_metrics(value):
|
||||
if isinstance(value, list):
|
||||
return [metric for child in value for metric in kernel_metrics(child)]
|
||||
if isinstance(value, dict):
|
||||
metrics = value.get("metrics", {})
|
||||
return ([metrics] if metrics.get("count", 0) else []) + [
|
||||
metric for child in value.values() for metric in kernel_metrics(child)
|
||||
]
|
||||
return []
|
||||
|
||||
try:
|
||||
with Worker._get_cudagraph_capture_context(worker):
|
||||
with proton.scope("capture_only"):
|
||||
capture_only()
|
||||
torch.accelerator.synchronize()
|
||||
with torch.cuda.graph(graph), proton.scope("captured_add"):
|
||||
captured_add()
|
||||
for run in range(2):
|
||||
wrapper.start()
|
||||
with wrapper.annotate_context_manager(f"replay_{run}"):
|
||||
graph.replay()
|
||||
wrapper.stop()
|
||||
(path,) = tmp_path.glob(f"*_run{run}.{output_format}")
|
||||
if output_format == "hatchet_msgpack":
|
||||
import msgpack
|
||||
|
||||
data = msgpack.unpackb(path.read_bytes())
|
||||
else:
|
||||
data = json.loads(path.read_text())
|
||||
serialized = json.dumps(data)
|
||||
assert "<captured_at>" in serialized
|
||||
assert "captured_add" in serialized
|
||||
assert "capture_only" not in serialized
|
||||
assert f"replay_{1 - run}" not in serialized
|
||||
metrics = kernel_metrics(data)
|
||||
assert sum(metric["count"] for metric in metrics) == 1
|
||||
assert sum(metric["time (ns)"] for metric in metrics) > 0
|
||||
torch.testing.assert_close(x, torch.full_like(x, 4))
|
||||
finally:
|
||||
wrapper.shutdown()
|
||||
assert not list(tmp_path.glob(".proton_cuda_graph_session*"))
|
||||
|
||||
@@ -76,6 +76,10 @@ class ProfilerConfig:
|
||||
"""Optional format passed to Proton when finalizing a profile. ``None``
|
||||
uses the default format for ``proton_data``."""
|
||||
|
||||
proton_graph_attribution: bool = False
|
||||
"""Observe CUDA graph capture so replayed kernels can be attributed.
|
||||
Requires Triton >= 3.7 and ``proton_data='tree'``."""
|
||||
|
||||
torch_profiler_with_stack: bool = True
|
||||
"""If `True`, enables stack tracing in the torch profiler. Enabled by default
|
||||
as it is useful for debugging. Can be disabled via
|
||||
@@ -199,6 +203,7 @@ class ProfilerConfig:
|
||||
("proton_mode", self.proton_mode, None),
|
||||
("proton_hook", self.proton_hook, None),
|
||||
("proton_output_format", self.proton_output_format, None),
|
||||
("proton_graph_attribution", self.proton_graph_attribution, False),
|
||||
)
|
||||
if value != default
|
||||
]
|
||||
@@ -220,6 +225,17 @@ class ProfilerConfig:
|
||||
|
||||
if self.profiler == "proton":
|
||||
output_format = self.proton_output_format
|
||||
if self.proton_graph_attribution and self.proton_data != "tree":
|
||||
raise ValueError("proton_graph_attribution requires proton_data='tree'")
|
||||
if (
|
||||
self.proton_graph_attribution
|
||||
and self.proton_mode
|
||||
and self.proton_mode.split(":", 1)[0].lower() == "periodic_flushing"
|
||||
):
|
||||
raise ValueError(
|
||||
"proton_graph_attribution is incompatible with periodic_flushing: "
|
||||
"both manage Proton data phases."
|
||||
)
|
||||
if output_format == "chrome_trace" and self.proton_data != "trace":
|
||||
raise ValueError("chrome_trace output requires proton_data='trace'")
|
||||
if (
|
||||
|
||||
+39
-12
@@ -1551,18 +1551,6 @@ class VllmConfig:
|
||||
self.compilation_config.mode = CompilationMode.NONE
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
if self.profiler_config.profiler == "proton":
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError(
|
||||
"The Proton profiler currently supports NVIDIA CUDA only"
|
||||
)
|
||||
if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
|
||||
raise ValueError(
|
||||
"The Proton profiler requires CUDA graphs to be disabled. "
|
||||
"Use --enforce-eager or set "
|
||||
"--compilation-config.cudagraph_mode=none."
|
||||
)
|
||||
|
||||
if os.environ.get("TORCH_COMPILE_DISABLE") == "1":
|
||||
logger.warning_once(
|
||||
"TORCH_COMPILE_DISABLE is set, disabling torch.compile. "
|
||||
@@ -1860,6 +1848,7 @@ class VllmConfig:
|
||||
else:
|
||||
self._validate_v1_model_runner()
|
||||
|
||||
self._validate_profiler_config()
|
||||
self._validate_batch_sharded_sampling()
|
||||
self._validate_adaptive_verification()
|
||||
|
||||
@@ -3042,6 +3031,44 @@ class VllmConfig:
|
||||
|
||||
return unsupported
|
||||
|
||||
def _validate_profiler_config(self) -> None:
|
||||
if self.profiler_config.profiler != "proton":
|
||||
return
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
raise ValueError("The Proton profiler currently supports NVIDIA CUDA only")
|
||||
if (
|
||||
self.profiler_config.proton_graph_attribution
|
||||
and not self.use_v2_model_runner
|
||||
):
|
||||
raise ValueError(
|
||||
"Proton CUDA graph attribution requires the V2 model runner."
|
||||
)
|
||||
|
||||
has_cuda_graphs = (
|
||||
self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
or (
|
||||
self.compilation_config.cudagraph_mm_encoder
|
||||
and not (self.model_config and self.model_config.enforce_eager)
|
||||
)
|
||||
)
|
||||
if not has_cuda_graphs:
|
||||
return
|
||||
mode = self.profiler_config.proton_mode
|
||||
if mode and mode.split(":", 1)[0].lower() == "pcsampling":
|
||||
raise ValueError(
|
||||
"Proton PC sampling requires CUDA graphs to be disabled. "
|
||||
"Use --enforce-eager."
|
||||
)
|
||||
if not self.profiler_config.proton_graph_attribution:
|
||||
raise ValueError(
|
||||
"Proton profiling with CUDA graphs requires "
|
||||
"proton_graph_attribution=True to capture replayed kernels. "
|
||||
"Enable attribution or use --enforce-eager."
|
||||
)
|
||||
|
||||
def _validate_v2_model_runner(self) -> None:
|
||||
"""Check for features not yet supported by the V2 model runner."""
|
||||
if not HAS_TRITON:
|
||||
|
||||
+138
-11
@@ -6,8 +6,9 @@ import inspect
|
||||
import json
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
|
||||
from glob import glob
|
||||
from typing import Literal
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -160,6 +161,15 @@ class WorkerProfiler(ABC):
|
||||
if self._running:
|
||||
self.stop()
|
||||
|
||||
@property
|
||||
def has_cuda_graph_session(self) -> bool:
|
||||
"""Whether a capture-time session is retained to attribute replays."""
|
||||
return False
|
||||
|
||||
def capture_cuda_graphs(self) -> AbstractContextManager[None]:
|
||||
"""Observe graph creation for backends that attribute replay activity."""
|
||||
return nullcontext()
|
||||
|
||||
def annotate_context_manager(self, name: str):
|
||||
"""Return a context manager to annotate profiler traces."""
|
||||
return nullcontext()
|
||||
@@ -387,6 +397,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
|
||||
self._mode = profiler_config.proton_mode
|
||||
self._hook = profiler_config.proton_hook
|
||||
self._output_format = profiler_config.proton_output_format
|
||||
self._graph_attribution = profiler_config.proton_graph_attribution
|
||||
self._triton_version_string = getattr(triton, "__version__", "unknown")
|
||||
try:
|
||||
self._triton_version = Version(self._triton_version_string)
|
||||
@@ -398,6 +409,13 @@ class ProtonProfilerWrapper(WorkerProfiler):
|
||||
# worker cannot overwrite profiles left by an earlier server process.
|
||||
self._instance_id = f"pid{os.getpid()}_{uuid4().hex}"
|
||||
self._run_id = 0
|
||||
self._graph_session = False
|
||||
self._phase = 0
|
||||
self._active_output_path: str | None = None
|
||||
self._session_storage_path = os.path.join(
|
||||
self._output_dir,
|
||||
f".proton_cuda_graph_session_{worker_name}_{self._instance_id}",
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Proton profiling enabled. Output will be saved under: %s",
|
||||
@@ -412,6 +430,10 @@ class ProtonProfilerWrapper(WorkerProfiler):
|
||||
)
|
||||
|
||||
def _validate_capabilities(self) -> None:
|
||||
if self._graph_attribution:
|
||||
self._require_triton_version(
|
||||
"CUDA graph attribution", _TRITON_PROTON_3_7_VERSION
|
||||
)
|
||||
if self._output_format is not None:
|
||||
parameters = inspect.signature(self._proton.finalize).parameters
|
||||
supports_output_format = "output_format" in parameters or any(
|
||||
@@ -428,7 +450,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
|
||||
self._require_triton_version(
|
||||
"hatchet_msgpack output", _TRITON_PROTON_3_7_VERSION
|
||||
)
|
||||
if self._mode and self._mode.split(":", 1)[0] == "periodic_flushing":
|
||||
if self._mode and self._mode.split(":", 1)[0].lower() == "periodic_flushing":
|
||||
self._require_triton_version(
|
||||
"periodic flushing", _TRITON_PROTON_3_7_VERSION
|
||||
)
|
||||
@@ -447,28 +469,112 @@ class ProtonProfilerWrapper(WorkerProfiler):
|
||||
raise RuntimeError("Proton did not create a profiling session")
|
||||
return session_id
|
||||
|
||||
@property
|
||||
def has_cuda_graph_session(self) -> bool:
|
||||
return self._graph_session
|
||||
|
||||
def set_output_name(self, worker_name: str) -> None:
|
||||
"""Set the next run's output name after startup graph capture."""
|
||||
if self._active:
|
||||
return
|
||||
self._output_path = os.path.join(self._output_dir, f"proton_{worker_name}")
|
||||
|
||||
@override
|
||||
@contextmanager
|
||||
def capture_cuda_graphs(self) -> Iterator[None]:
|
||||
"""Keep a Proton session active while vLLM captures CUDA graphs."""
|
||||
if not self._graph_attribution:
|
||||
yield
|
||||
return
|
||||
if self._session_id is None:
|
||||
self._session_id = self._create_session(self._session_storage_path)
|
||||
else:
|
||||
self._proton.activate(session=self._session_id)
|
||||
self._graph_session = True
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
captured_phase = self._phase
|
||||
try:
|
||||
self._phase = self._proton.data.advance_phase(self._session_id)
|
||||
finally:
|
||||
self._proton.deactivate(session=self._session_id, flushing=True)
|
||||
self._proton.data.clear(self._session_id, captured_phase)
|
||||
|
||||
@override
|
||||
def _start(self) -> None:
|
||||
output_path = f"{self._output_path}_{self._instance_id}_run{self._run_id}"
|
||||
self._session_id = self._create_session(output_path)
|
||||
self._active_output_path = (
|
||||
f"{self._output_path}_{self._instance_id}_run{self._run_id}"
|
||||
)
|
||||
self._run_id += 1
|
||||
if self._graph_session:
|
||||
assert self._session_id is not None
|
||||
self._proton.activate(session=self._session_id)
|
||||
else:
|
||||
self._session_id = self._create_session(self._active_output_path)
|
||||
|
||||
def _write_graph_phase(self, phase: int) -> None:
|
||||
assert self._active_output_path is not None
|
||||
output_format = self._output_format or "hatchet"
|
||||
output_path = f"{self._active_output_path}.{output_format}"
|
||||
if output_format == "hatchet_msgpack":
|
||||
with open(output_path, "wb") as output_file:
|
||||
output_file.write(
|
||||
self._proton.data.get_msgpack(self._session_id, phase)
|
||||
)
|
||||
else:
|
||||
with open(output_path, "w", encoding="utf-8") as output_file:
|
||||
json.dump(self._proton.data.get(self._session_id, phase), output_file)
|
||||
|
||||
def _finalize_session(self, session_id: int) -> None:
|
||||
if self._output_format is None:
|
||||
self._proton.finalize(session=session_id)
|
||||
else:
|
||||
self._proton.finalize(session=session_id, output_format=self._output_format)
|
||||
|
||||
@override
|
||||
def _stop(self) -> None:
|
||||
assert self._session_id is not None
|
||||
session_id = self._session_id
|
||||
if self._graph_session:
|
||||
completed_phase = self._phase
|
||||
try:
|
||||
self._phase = self._proton.data.advance_phase(session_id)
|
||||
finally:
|
||||
self._proton.deactivate(session=session_id, flushing=True)
|
||||
try:
|
||||
self._write_graph_phase(completed_phase)
|
||||
finally:
|
||||
self._proton.data.clear(session_id, completed_phase)
|
||||
self._active_output_path = None
|
||||
return
|
||||
|
||||
try:
|
||||
self._proton.deactivate(session=session_id)
|
||||
finally:
|
||||
try:
|
||||
if self._output_format is None:
|
||||
self._proton.finalize(session=session_id)
|
||||
else:
|
||||
self._proton.finalize(
|
||||
session=session_id, output_format=self._output_format
|
||||
)
|
||||
self._finalize_session(session_id)
|
||||
finally:
|
||||
self._session_id = None
|
||||
self._active_output_path = None
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
super().shutdown()
|
||||
if self._graph_session and self._session_id is not None:
|
||||
session_id = self._session_id
|
||||
self._session_id = None
|
||||
try:
|
||||
self._finalize_session(session_id)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to finalize Proton CUDA graph session during shutdown."
|
||||
)
|
||||
finally:
|
||||
for output_path in glob(f"{self._session_storage_path}.*"):
|
||||
with suppress(FileNotFoundError):
|
||||
os.remove(output_path)
|
||||
|
||||
@override
|
||||
def annotate_context_manager(self, name: str):
|
||||
@@ -496,3 +602,24 @@ class CudaProfilerWrapper(WorkerProfiler):
|
||||
@override
|
||||
def annotate_context_manager(self, name: str):
|
||||
return torch.cuda.nvtx.range(name)
|
||||
|
||||
|
||||
def create_graph_capture_profiler(
|
||||
profiler_config: ProfilerConfig, global_rank: int
|
||||
) -> WorkerProfiler | None:
|
||||
"""Create a profiler to observe CUDA graph capture, if configured.
|
||||
|
||||
Applies only to backends that attribute graph replay activity which
|
||||
need a session around capture (Proton with ``proton_graph_attribution``).
|
||||
"""
|
||||
if (
|
||||
profiler_config.profiler == "proton"
|
||||
and profiler_config.proton_graph_attribution
|
||||
):
|
||||
from vllm.distributed.utils import get_worker_rank_suffix
|
||||
|
||||
return ProtonProfilerWrapper(
|
||||
profiler_config,
|
||||
worker_name=get_worker_rank_suffix(global_rank=global_rank),
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -987,6 +987,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
"""Estimate the GPU memory required to capture CUDA graphs."""
|
||||
return _profile_cudagraph_memory(self)
|
||||
|
||||
def needs_cudagraph_capture(self) -> bool:
|
||||
"""Whether capture_model() has any CUDA graphs left to capture."""
|
||||
return (
|
||||
self.model_state.supports_mm_inputs
|
||||
and self.model_state.encoder_runner.has_cudagraph()
|
||||
) or (
|
||||
self.cudagraph_manager is not None
|
||||
and self.cudagraph_manager.needs_capture()
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def capture_model(self, *, profile_only: bool = False) -> int:
|
||||
assert self.cudagraph_manager is not None
|
||||
|
||||
@@ -9,7 +9,7 @@ from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from datetime import timedelta
|
||||
from types import NoneType
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
import regex as re
|
||||
@@ -64,6 +64,7 @@ from vllm.profiler.wrapper import (
|
||||
CudaProfilerWrapper,
|
||||
ProtonProfilerWrapper,
|
||||
TorchProfilerWrapper,
|
||||
create_graph_capture_profiler,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -142,6 +143,7 @@ def maybe_rocm_profiling_fallback(profile_result: MemoryProfilingResult) -> int
|
||||
if TYPE_CHECKING:
|
||||
from vllm.device_allocator.sleep_mode_backend import SleepModeBackend
|
||||
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as GPUModelRunnerV2
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
|
||||
@@ -810,7 +812,8 @@ class Worker(WorkerBase):
|
||||
|
||||
cuda_graph_memory_bytes = 0
|
||||
if not self.model_config.enforce_eager:
|
||||
cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
with self._get_cudagraph_capture_context():
|
||||
cuda_graph_memory_bytes = self.model_runner.capture_model()
|
||||
|
||||
# Compare actual vs estimated CUDA graph memory (if we did profiling)
|
||||
if (
|
||||
@@ -949,6 +952,21 @@ class Worker(WorkerBase):
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
)
|
||||
|
||||
def _get_cudagraph_capture_context(self) -> AbstractContextManager[None]:
|
||||
"""Let the configured profiler observe CUDA graph capture."""
|
||||
if not self.use_v2_model_runner:
|
||||
return nullcontext()
|
||||
if self.profiler is None:
|
||||
model_runner = cast("GPUModelRunnerV2", self.model_runner)
|
||||
if not model_runner.needs_cudagraph_capture():
|
||||
return nullcontext()
|
||||
self.profiler = create_graph_capture_profiler(
|
||||
self.profiler_config, global_rank=self.rank
|
||||
)
|
||||
if self.profiler is None:
|
||||
return nullcontext()
|
||||
return self.profiler.capture_cuda_graphs()
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
self.model_runner.reset_mm_cache()
|
||||
|
||||
@@ -1248,6 +1266,9 @@ class Worker(WorkerBase):
|
||||
else:
|
||||
trace_name = rank_suffix
|
||||
|
||||
if profiler_type == "proton" and self.profiler is not None:
|
||||
self.profiler.set_output_name(trace_name)
|
||||
|
||||
# Create the profiler wrapper only on the first start call
|
||||
if self.profiler is None:
|
||||
if profiler_type == "torch":
|
||||
@@ -1284,7 +1305,9 @@ class Worker(WorkerBase):
|
||||
try:
|
||||
self.profiler.stop()
|
||||
finally:
|
||||
if self.profiler_config.profiler == "proton":
|
||||
if self.profiler_config.profiler == "proton" and not (
|
||||
self.profiler.has_cuda_graph_session
|
||||
):
|
||||
# Proton output names are fixed when the wrapper is constructed.
|
||||
# Recreate it so the next profile_prefix is honored.
|
||||
self.profiler = None
|
||||
|
||||
@@ -70,6 +70,9 @@ class MMEncoderModelRunner(GPUModelRunner):
|
||||
def capture_model(self, *, profile_only: bool = False) -> int:
|
||||
return 0
|
||||
|
||||
def needs_cudagraph_capture(self) -> bool:
|
||||
return False
|
||||
|
||||
def _dummy_run(
|
||||
self, *args: Any, **kwargs: Any
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user