[Profiler] Add Proton CUDA graph attribution for MRV2 (#51084)

Signed-off-by: Luossu <zhluosuu@outlook.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
鐘天楽
2026-09-14 19:02:07 +00:00
committed by GitHub
co-authored by Nick Hill
parent 5372e72a98
commit 0a5747d410
8 changed files with 657 additions and 45 deletions
+31 -13
View File
@@ -90,18 +90,18 @@ $ curl -X POST http://localhost:8000/stop_profile
is Triton's GPU profiler. It can collect a low-overhead aggregate tree or a
Chrome trace and works through the same vLLM profiling controls as the PyTorch
and CUDA profilers. Proton currently supports NVIDIA GPUs through CUPTI and
requires eager execution.
supports CUDA graph attribution.
Start a server with a local output directory:
Start a server with a local output directory and graph attribution:
```bash
vllm serve meta-llama/Llama-3.1-8B-Instruct \
--enforce-eager \
--profiler-config '{
"profiler": "proton",
"proton_profiler_dir": "./proton_profile",
"proton_output_format": "hatchet",
"proton_hook": "triton"
"proton_hook": "triton",
"proton_graph_attribution": true
}'
```
@@ -111,7 +111,7 @@ rank-qualified output name, such as
`proton_dp0_pp0_tp0_dcp0_ep0_rank0_pid1234_0123456789abcdef0123456789abcdef_run0.hatchet`,
so distributed workers, restarted servers, and repeated profiling runs do not
overwrite one another. A `profile_prefix` is included when supplied. Each
profile is finalized by `/stop_profile` and is ready to inspect immediately.
profile is written by `/stop_profile` and is ready to inspect immediately.
The Proton-specific options are:
@@ -121,21 +121,39 @@ The Proton-specific options are:
- `proton_mode`: an optional backend mode string
- `proton_hook`: `triton` to record Triton launch metadata, or unset
- `proton_output_format`: `hatchet`, `hatchet_msgpack`, `chrome_trace`, or unset
- `proton_graph_attribution`: observe CUDA graph capture for replay attribution;
disabled by default and requires `proton_data: "tree"`
`hatchet` and `hatchet_msgpack` require `proton_data: "tree"`, while
`chrome_trace` requires `proton_data: "trace"`. CUDA graph profiling is not yet
supported, so `--enforce-eager` is required for every Proton mode.
`chrome_trace` requires `proton_data: "trace"`.
Automatic backend selection is recommended. vLLM currently supports Proton's
`cupti` backend on NVIDIA GPUs. ROCm support is not yet available. vLLM does not
expose Proton's experimental instrumentation backend because current upstream
Triton builds can produce profiles without timing metrics. Backend-specific
modes, including `pcsampling`, can be selected with `proton_mode`.
can produce profiles without timing metrics. When `proton_graph_attribution` is
enabled, Proton observes vLLM's CUDA graph capture with the configured profiling
session active, then deactivates that same session until profiling starts. This
lets later profiles attribute replayed kernels without retaining model-startup
activity. Backend-specific modes can be selected with `proton_mode`;
`pcsampling` synchronizes the CUDA context and therefore requires
`--enforce-eager`. When CUDA graphs are enabled (including encoder graphs),
Proton requires `proton_graph_attribution: true` to collect replayed kernels.
For Chrome traces, disable CUDA graphs with `--enforce-eager`.
Triton 3.6 supports explicit `hatchet` and `chrome_trace` output. The
`hatchet_msgpack` format and `periodic_flushing` mode require Triton 3.7 or
newer; vLLM rejects these options with the detected version before starting a
session.
CUDA graph-attributed profiles support repeated `start_profile`/`stop_profile`
runs. Each stop flushes and writes one tree-data phase while preserving the
graph-aware session. Without `proton_graph_attribution`, each `stop_profile`
instead finalizes and writes an independent Proton session.
CUDA graph attribution requires Triton 3.7 or newer. It uses the phase data API
to discard graph-capture activity and separate profiling runs. The `hatchet_msgpack`
output format and `periodic_flushing` mode also require Triton 3.7 or newer.
`periodic_flushing` cannot be combined with graph attribution because both
manage the session's data phases. Ordinary Proton profiling remains available
with Triton 3.6.
Graph attribution retains capture metadata for the worker lifetime. With eager
execution or no graphs to capture, profiling uses ordinary independent sessions.
Inspect tree profiles with:
+394 -6
View File
@@ -350,17 +350,24 @@ def test_profiler_entered_during_capture():
def make_proton(session_id: int | None = 7):
data = SimpleNamespace(
advance_phase=Mock(side_effect=range(1, 100)),
clear=Mock(),
get=Mock(return_value={"traceEvents": []}),
get_msgpack=Mock(return_value=b"profile"),
)
return SimpleNamespace(
start=Mock(return_value=session_id),
activate=Mock(),
deactivate=Mock(),
finalize=Mock(),
scope=Mock(return_value=nullcontext()),
data=data,
)
def make_proton_wrapper(
tmp_path, proton=None, triton_version="3.6.0", **config_overrides
tmp_path, proton=None, triton_version="3.7.0", **config_overrides
):
proton = proton or make_proton()
config = ProfilerConfig(
@@ -415,6 +422,13 @@ class TestProtonConfig:
},
"requires proton_data",
),
(
{
"proton_data": "trace",
"proton_graph_attribution": True,
},
"requires proton_data='tree'",
),
],
)
def test_rejects_invalid_option_combinations(self, tmp_path, options, message):
@@ -450,6 +464,7 @@ class TestProtonConfig:
("proton_mode", "pcsampling"),
("proton_hook", "triton"),
("proton_output_format", "chrome_trace"),
("proton_graph_attribution", True),
],
)
def test_rejects_proton_options_for_other_profilers(self, field, value):
@@ -478,15 +493,101 @@ class TestProtonConfig:
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.NONE),
)
def test_rejects_proton_when_cuda_graphs_are_enabled(self, tmp_path):
with pytest.raises(ValueError, match="requires CUDA graphs to be disabled"):
@pytest.mark.parametrize("encoder_only", [False, True])
@pytest.mark.parametrize("attribution", [False, True])
def test_cuda_graphs_require_attribution(self, tmp_path, encoder_only, attribution):
# Encoder graphs are independent of the decoder's cudagraph_mode.
expected = (
nullcontext()
if attribution
else pytest.raises(
ValueError, match="requires proton_graph_attribution=True"
)
)
with expected:
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=attribution,
),
compilation_config=CompilationConfig(
cudagraph_mode=(
CUDAGraphMode.NONE if encoder_only else CUDAGraphMode.FULL
),
cudagraph_mm_encoder=encoder_only,
),
)
def test_validates_default_cuda_graph_mode_after_resolution(self, tmp_path):
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton", proton_profiler_dir=str(tmp_path)
),
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.PIECEWISE
)
@pytest.mark.parametrize(
"mode",
[
"periodic_flushing",
"periodic_flushing:format=hatchet",
"PERIODIC_FLUSHING:format=hatchet",
],
)
def test_graph_attribution_rejects_periodic_flushing(self, tmp_path, mode):
# Reject before Proton's native phase manager can abort the worker.
with pytest.raises(ValueError, match="incompatible with periodic_flushing"):
ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
proton_mode=mode,
)
@pytest.mark.parametrize("attribution", [False, True])
@pytest.mark.parametrize(
"mode", ["pcsampling", "pcsampling:interval=100", "PcSampling:interval=100"]
)
@pytest.mark.parametrize("encoder_only", [False, True])
def test_pcsampling_requires_graphs_disabled(
self, tmp_path, attribution, mode, encoder_only
):
with pytest.raises(ValueError, match="PC sampling requires CUDA graphs"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=attribution,
proton_mode=mode,
),
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.NONE
if encoder_only
else CUDAGraphMode.FULL,
cudagraph_mm_encoder=encoder_only,
),
)
def test_ordinary_proton_keeps_mrv1_graph_restriction(self, tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
with pytest.raises(ValueError, match="requires proton_graph_attribution=True"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton", proton_profiler_dir=str(tmp_path)
),
compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL),
)
def test_graph_attribution_rejects_mrv1(self, tmp_path, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
with pytest.raises(ValueError, match="requires the V2 model runner"):
VllmConfig(
profiler_config=ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
)
)
@@ -564,7 +665,20 @@ class TestProtonProfilerWrapper:
)
def test_newer_features_reject_triton_3_6(self, tmp_path, option, value, feature):
with pytest.raises(RuntimeError, match=feature):
make_proton_wrapper(tmp_path, **{option: value})
make_proton_wrapper(tmp_path, triton_version="3.6.0", **{option: value})
@pytest.mark.parametrize("version", ["3.6.0", "unknown"])
def test_graph_attribution_requires_phase_api(self, tmp_path, version):
with pytest.raises(RuntimeError, match="requires Triton >= 3.7"):
make_proton_wrapper(
tmp_path, triton_version=version, proton_graph_attribution=True
)
def test_ordinary_profiling_still_supports_triton_3_6(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, triton_version="3.6.0")
wrapper.start()
wrapper.stop()
proton.finalize.assert_called_once_with(session=7)
@pytest.mark.parametrize(
("option", "value"),
@@ -583,6 +697,125 @@ class TestProtonProfilerWrapper:
with pytest.raises(RuntimeError, match="does not support selecting"):
make_proton_wrapper(tmp_path, proton, proton_output_format="hatchet")
def test_cuda_graph_tree_phase_is_written_at_stop(self, tmp_path):
wrapper, proton = make_proton_wrapper(
tmp_path,
proton_context="python",
proton_graph_attribution=True,
)
with wrapper.capture_cuda_graphs():
proton.start.assert_called_once()
proton.deactivate.assert_not_called()
capture_args = proton.start.call_args.kwargs
assert capture_args["context"] == "python"
assert capture_args["data"] == "tree"
proton.data.advance_phase.assert_called_once_with(7)
proton.deactivate.assert_called_once_with(session=7, flushing=True)
proton.data.clear.assert_called_once_with(7, 0)
wrapper.start()
wrapper.stop()
wrapper.start()
wrapper.stop()
assert proton.data.get.call_args_list == [call(7, 1), call(7, 2)]
proton.data.clear.assert_has_calls([call(7, 0), call(7, 1), call(7, 2)])
output_names = sorted(tmp_path.glob("proton_rank_3_*.hatchet"))
assert len(output_names) == 2
assert output_names[0].name.endswith("_run0.hatchet")
assert output_names[1].name.endswith("_run1.hatchet")
def test_capture_is_noop_without_opt_in(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path)
with wrapper.capture_cuda_graphs():
pass
proton.start.assert_not_called()
@pytest.mark.parametrize("delay", [0, 2])
def test_duplicate_start_preserves_output_prefix(self, tmp_path, delay):
wrapper, proton = make_proton_wrapper(
tmp_path, proton_graph_attribution=True, delay_iterations=delay
)
with wrapper.capture_cuda_graphs():
pass
wrapper.set_output_name("first")
wrapper.start()
wrapper.set_output_name("duplicate")
wrapper.start()
for _ in range(delay):
wrapper.step()
wrapper.stop()
assert len(list(tmp_path.glob("proton_first_*.hatchet"))) == 1
assert not list(tmp_path.glob("proton_duplicate_*"))
wrapper.set_output_name("second")
wrapper.start()
for _ in range(delay):
wrapper.step()
wrapper.stop()
assert len(list(tmp_path.glob("proton_second_*.hatchet"))) == 1
def test_failed_export_clears_activity_and_allows_next_interval(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
proton.data.get.side_effect = [OSError("disk full"), []]
wrapper.start()
wrapper.stop()
proton.data.clear.assert_called_with(7, 1)
wrapper.start()
wrapper.stop()
proton.data.clear.assert_called_with(7, 2)
assert len(list(tmp_path.glob("*_run1.hatchet"))) == 1
def test_cuda_graph_context_deactivates_after_capture_error(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with (
pytest.raises(RuntimeError, match="capture failed"),
wrapper.capture_cuda_graphs(),
):
raise RuntimeError("capture failed")
proton.deactivate.assert_called_once_with(session=7, flushing=True)
def test_cuda_graph_capture_deactivates_when_phase_advance_fails(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
with (
pytest.raises(RuntimeError, match="advance failed"),
wrapper.capture_cuda_graphs(),
):
pass
proton.deactivate.assert_called_once_with(session=7, flushing=True)
proton.data.clear.assert_not_called()
def test_cuda_graph_stop_deactivates_when_phase_advance_fails(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
proton.data.clear.reset_mock()
proton.data.advance_phase.side_effect = RuntimeError("advance failed")
wrapper._start()
with pytest.raises(RuntimeError, match="advance failed"):
wrapper._stop()
proton.deactivate.assert_called_with(session=7, flushing=True)
proton.data.clear.assert_not_called()
def test_shutdown_finalizes_cuda_graph_capture_session(self, tmp_path):
wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True)
with wrapper.capture_cuda_graphs():
pass
wrapper.shutdown()
wrapper.shutdown()
proton.finalize.assert_called_once_with(session=7)
def test_missing_proton_has_actionable_error(self, tmp_path):
config = ProfilerConfig(profiler="proton", proton_profiler_dir=str(tmp_path))
with (
@@ -638,6 +871,7 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
),
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
):
wrapper.return_value.has_cuda_graph_session = False
Worker.profile(worker, profile_prefix="first")
Worker.profile(worker, is_start=False)
Worker.profile(worker, profile_prefix="second")
@@ -647,3 +881,157 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run():
call(worker.profiler_config, worker_name="second_rank1"),
]
assert wrapper.return_value.start.call_count == 2
@_requires_cuda_for_proton
def test_gpu_worker_reuses_cuda_graph_proton_session():
worker = MagicMock()
worker.rank = 1
worker.profiler = MagicMock(spec=ProtonProfilerWrapper)
worker.profiler.has_cuda_graph_session = True
worker.profiler_config.profiler = "proton"
with patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank1",
):
Worker.profile(worker, profile_prefix="first")
Worker.profile(worker, is_start=False)
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
worker.profiler.start.assert_called_once_with()
worker.profiler.stop.assert_called_once_with()
@_requires_cuda_for_proton
@pytest.mark.parametrize("runner", ["attribution_off", "v1", "no_capture"])
def test_proton_not_initialized_without_capture(runner):
worker = MagicMock()
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config.proton_graph_attribution = runner != "attribution_off"
worker.use_v2_model_runner = runner != "v1"
worker.model_runner.needs_cudagraph_capture.return_value = runner != "no_capture"
context = Worker._get_cudagraph_capture_context(worker)
assert worker.profiler is None
with context:
pass
@_requires_cuda_for_proton
def test_proton_initializes_before_cuda_graph_capture():
class FakeProtonProfiler:
def __init__(self, config, worker_name):
self.config = config
self.worker_name = worker_name
self.capture_context = nullcontext()
def capture_cuda_graphs(self):
return self.capture_context
worker = MagicMock()
worker.rank = 2
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config.proton_graph_attribution = True
worker.use_v2_model_runner = True
worker.model_runner.needs_cudagraph_capture.return_value = True
with (
patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank2",
),
patch(
"vllm.profiler.wrapper.ProtonProfilerWrapper",
FakeProtonProfiler,
),
):
context = Worker._get_cudagraph_capture_context(worker)
assert worker.profiler.config is worker.profiler_config
assert worker.profiler.worker_name == "rank2"
assert context is worker.profiler.capture_context
@_requires_cuda_for_proton
@pytest.mark.parametrize("context", ["shadow", "python"])
@pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"])
def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format):
"""Both intervals contain replay kernels, without capture-only activity."""
import json
import torch
import triton
import triton.profiler as proton
from packaging.version import Version
if Version(triton.__version__) < Version("3.7"):
pytest.skip("Graph attribution requires Triton >= 3.7")
wrapper = ProtonProfilerWrapper(
ProfilerConfig(
profiler="proton",
proton_profiler_dir=str(tmp_path),
proton_graph_attribution=True,
proton_context=context,
proton_output_format=output_format,
),
worker_name="gpu",
)
worker = SimpleNamespace(
profiler=wrapper,
use_v2_model_runner=True,
)
x = torch.ones(1024, device="cuda")
graph = torch.cuda.CUDAGraph()
def capture_only():
x.add_(1)
def captured_add():
x.add_(1)
def kernel_metrics(value):
if isinstance(value, list):
return [metric for child in value for metric in kernel_metrics(child)]
if isinstance(value, dict):
metrics = value.get("metrics", {})
return ([metrics] if metrics.get("count", 0) else []) + [
metric for child in value.values() for metric in kernel_metrics(child)
]
return []
try:
with Worker._get_cudagraph_capture_context(worker):
with proton.scope("capture_only"):
capture_only()
torch.accelerator.synchronize()
with torch.cuda.graph(graph), proton.scope("captured_add"):
captured_add()
for run in range(2):
wrapper.start()
with wrapper.annotate_context_manager(f"replay_{run}"):
graph.replay()
wrapper.stop()
(path,) = tmp_path.glob(f"*_run{run}.{output_format}")
if output_format == "hatchet_msgpack":
import msgpack
data = msgpack.unpackb(path.read_bytes())
else:
data = json.loads(path.read_text())
serialized = json.dumps(data)
assert "<captured_at>" in serialized
assert "captured_add" in serialized
assert "capture_only" not in serialized
assert f"replay_{1 - run}" not in serialized
metrics = kernel_metrics(data)
assert sum(metric["count"] for metric in metrics) == 1
assert sum(metric["time (ns)"] for metric in metrics) > 0
torch.testing.assert_close(x, torch.full_like(x, 4))
finally:
wrapper.shutdown()
assert not list(tmp_path.glob(".proton_cuda_graph_session*"))
+16
View File
@@ -76,6 +76,10 @@ class ProfilerConfig:
"""Optional format passed to Proton when finalizing a profile. ``None``
uses the default format for ``proton_data``."""
proton_graph_attribution: bool = False
"""Observe CUDA graph capture so replayed kernels can be attributed.
Requires Triton >= 3.7 and ``proton_data='tree'``."""
torch_profiler_with_stack: bool = True
"""If `True`, enables stack tracing in the torch profiler. Enabled by default
as it is useful for debugging. Can be disabled via
@@ -199,6 +203,7 @@ class ProfilerConfig:
("proton_mode", self.proton_mode, None),
("proton_hook", self.proton_hook, None),
("proton_output_format", self.proton_output_format, None),
("proton_graph_attribution", self.proton_graph_attribution, False),
)
if value != default
]
@@ -220,6 +225,17 @@ class ProfilerConfig:
if self.profiler == "proton":
output_format = self.proton_output_format
if self.proton_graph_attribution and self.proton_data != "tree":
raise ValueError("proton_graph_attribution requires proton_data='tree'")
if (
self.proton_graph_attribution
and self.proton_mode
and self.proton_mode.split(":", 1)[0].lower() == "periodic_flushing"
):
raise ValueError(
"proton_graph_attribution is incompatible with periodic_flushing: "
"both manage Proton data phases."
)
if output_format == "chrome_trace" and self.proton_data != "trace":
raise ValueError("chrome_trace output requires proton_data='trace'")
if (
+39 -12
View File
@@ -1551,18 +1551,6 @@ class VllmConfig:
self.compilation_config.mode = CompilationMode.NONE
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
if self.profiler_config.profiler == "proton":
if not current_platform.is_cuda():
raise ValueError(
"The Proton profiler currently supports NVIDIA CUDA only"
)
if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE:
raise ValueError(
"The Proton profiler requires CUDA graphs to be disabled. "
"Use --enforce-eager or set "
"--compilation-config.cudagraph_mode=none."
)
if os.environ.get("TORCH_COMPILE_DISABLE") == "1":
logger.warning_once(
"TORCH_COMPILE_DISABLE is set, disabling torch.compile. "
@@ -1860,6 +1848,7 @@ class VllmConfig:
else:
self._validate_v1_model_runner()
self._validate_profiler_config()
self._validate_batch_sharded_sampling()
self._validate_adaptive_verification()
@@ -3042,6 +3031,44 @@ class VllmConfig:
return unsupported
def _validate_profiler_config(self) -> None:
if self.profiler_config.profiler != "proton":
return
from vllm.platforms import current_platform
if not current_platform.is_cuda():
raise ValueError("The Proton profiler currently supports NVIDIA CUDA only")
if (
self.profiler_config.proton_graph_attribution
and not self.use_v2_model_runner
):
raise ValueError(
"Proton CUDA graph attribution requires the V2 model runner."
)
has_cuda_graphs = (
self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
or (
self.compilation_config.cudagraph_mm_encoder
and not (self.model_config and self.model_config.enforce_eager)
)
)
if not has_cuda_graphs:
return
mode = self.profiler_config.proton_mode
if mode and mode.split(":", 1)[0].lower() == "pcsampling":
raise ValueError(
"Proton PC sampling requires CUDA graphs to be disabled. "
"Use --enforce-eager."
)
if not self.profiler_config.proton_graph_attribution:
raise ValueError(
"Proton profiling with CUDA graphs requires "
"proton_graph_attribution=True to capture replayed kernels. "
"Enable attribution or use --enforce-eager."
)
def _validate_v2_model_runner(self) -> None:
"""Check for features not yet supported by the V2 model runner."""
if not HAS_TRITON:
+138 -11
View File
@@ -6,8 +6,9 @@ import inspect
import json
import os
from abc import ABC, abstractmethod
from collections.abc import Callable
from contextlib import nullcontext
from collections.abc import Callable, Iterator
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
from glob import glob
from typing import Literal
from uuid import uuid4
@@ -160,6 +161,15 @@ class WorkerProfiler(ABC):
if self._running:
self.stop()
@property
def has_cuda_graph_session(self) -> bool:
"""Whether a capture-time session is retained to attribute replays."""
return False
def capture_cuda_graphs(self) -> AbstractContextManager[None]:
"""Observe graph creation for backends that attribute replay activity."""
return nullcontext()
def annotate_context_manager(self, name: str):
"""Return a context manager to annotate profiler traces."""
return nullcontext()
@@ -387,6 +397,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
self._mode = profiler_config.proton_mode
self._hook = profiler_config.proton_hook
self._output_format = profiler_config.proton_output_format
self._graph_attribution = profiler_config.proton_graph_attribution
self._triton_version_string = getattr(triton, "__version__", "unknown")
try:
self._triton_version = Version(self._triton_version_string)
@@ -398,6 +409,13 @@ class ProtonProfilerWrapper(WorkerProfiler):
# worker cannot overwrite profiles left by an earlier server process.
self._instance_id = f"pid{os.getpid()}_{uuid4().hex}"
self._run_id = 0
self._graph_session = False
self._phase = 0
self._active_output_path: str | None = None
self._session_storage_path = os.path.join(
self._output_dir,
f".proton_cuda_graph_session_{worker_name}_{self._instance_id}",
)
logger.info_once(
"Proton profiling enabled. Output will be saved under: %s",
@@ -412,6 +430,10 @@ class ProtonProfilerWrapper(WorkerProfiler):
)
def _validate_capabilities(self) -> None:
if self._graph_attribution:
self._require_triton_version(
"CUDA graph attribution", _TRITON_PROTON_3_7_VERSION
)
if self._output_format is not None:
parameters = inspect.signature(self._proton.finalize).parameters
supports_output_format = "output_format" in parameters or any(
@@ -428,7 +450,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
self._require_triton_version(
"hatchet_msgpack output", _TRITON_PROTON_3_7_VERSION
)
if self._mode and self._mode.split(":", 1)[0] == "periodic_flushing":
if self._mode and self._mode.split(":", 1)[0].lower() == "periodic_flushing":
self._require_triton_version(
"periodic flushing", _TRITON_PROTON_3_7_VERSION
)
@@ -447,28 +469,112 @@ class ProtonProfilerWrapper(WorkerProfiler):
raise RuntimeError("Proton did not create a profiling session")
return session_id
@property
def has_cuda_graph_session(self) -> bool:
return self._graph_session
def set_output_name(self, worker_name: str) -> None:
"""Set the next run's output name after startup graph capture."""
if self._active:
return
self._output_path = os.path.join(self._output_dir, f"proton_{worker_name}")
@override
@contextmanager
def capture_cuda_graphs(self) -> Iterator[None]:
"""Keep a Proton session active while vLLM captures CUDA graphs."""
if not self._graph_attribution:
yield
return
if self._session_id is None:
self._session_id = self._create_session(self._session_storage_path)
else:
self._proton.activate(session=self._session_id)
self._graph_session = True
try:
yield
finally:
captured_phase = self._phase
try:
self._phase = self._proton.data.advance_phase(self._session_id)
finally:
self._proton.deactivate(session=self._session_id, flushing=True)
self._proton.data.clear(self._session_id, captured_phase)
@override
def _start(self) -> None:
output_path = f"{self._output_path}_{self._instance_id}_run{self._run_id}"
self._session_id = self._create_session(output_path)
self._active_output_path = (
f"{self._output_path}_{self._instance_id}_run{self._run_id}"
)
self._run_id += 1
if self._graph_session:
assert self._session_id is not None
self._proton.activate(session=self._session_id)
else:
self._session_id = self._create_session(self._active_output_path)
def _write_graph_phase(self, phase: int) -> None:
assert self._active_output_path is not None
output_format = self._output_format or "hatchet"
output_path = f"{self._active_output_path}.{output_format}"
if output_format == "hatchet_msgpack":
with open(output_path, "wb") as output_file:
output_file.write(
self._proton.data.get_msgpack(self._session_id, phase)
)
else:
with open(output_path, "w", encoding="utf-8") as output_file:
json.dump(self._proton.data.get(self._session_id, phase), output_file)
def _finalize_session(self, session_id: int) -> None:
if self._output_format is None:
self._proton.finalize(session=session_id)
else:
self._proton.finalize(session=session_id, output_format=self._output_format)
@override
def _stop(self) -> None:
assert self._session_id is not None
session_id = self._session_id
if self._graph_session:
completed_phase = self._phase
try:
self._phase = self._proton.data.advance_phase(session_id)
finally:
self._proton.deactivate(session=session_id, flushing=True)
try:
self._write_graph_phase(completed_phase)
finally:
self._proton.data.clear(session_id, completed_phase)
self._active_output_path = None
return
try:
self._proton.deactivate(session=session_id)
finally:
try:
if self._output_format is None:
self._proton.finalize(session=session_id)
else:
self._proton.finalize(
session=session_id, output_format=self._output_format
)
self._finalize_session(session_id)
finally:
self._session_id = None
self._active_output_path = None
@override
def shutdown(self) -> None:
super().shutdown()
if self._graph_session and self._session_id is not None:
session_id = self._session_id
self._session_id = None
try:
self._finalize_session(session_id)
except Exception:
logger.exception(
"Failed to finalize Proton CUDA graph session during shutdown."
)
finally:
for output_path in glob(f"{self._session_storage_path}.*"):
with suppress(FileNotFoundError):
os.remove(output_path)
@override
def annotate_context_manager(self, name: str):
@@ -496,3 +602,24 @@ class CudaProfilerWrapper(WorkerProfiler):
@override
def annotate_context_manager(self, name: str):
return torch.cuda.nvtx.range(name)
def create_graph_capture_profiler(
profiler_config: ProfilerConfig, global_rank: int
) -> WorkerProfiler | None:
"""Create a profiler to observe CUDA graph capture, if configured.
Applies only to backends that attribute graph replay activity which
need a session around capture (Proton with ``proton_graph_attribution``).
"""
if (
profiler_config.profiler == "proton"
and profiler_config.proton_graph_attribution
):
from vllm.distributed.utils import get_worker_rank_suffix
return ProtonProfilerWrapper(
profiler_config,
worker_name=get_worker_rank_suffix(global_rank=global_rank),
)
return None
+10
View File
@@ -987,6 +987,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"""Estimate the GPU memory required to capture CUDA graphs."""
return _profile_cudagraph_memory(self)
def needs_cudagraph_capture(self) -> bool:
"""Whether capture_model() has any CUDA graphs left to capture."""
return (
self.model_state.supports_mm_inputs
and self.model_state.encoder_runner.has_cudagraph()
) or (
self.cudagraph_manager is not None
and self.cudagraph_manager.needs_capture()
)
@torch.inference_mode()
def capture_model(self, *, profile_only: bool = False) -> int:
assert self.cudagraph_manager is not None
+26 -3
View File
@@ -9,7 +9,7 @@ from collections.abc import Callable
from contextlib import AbstractContextManager, contextmanager, nullcontext
from datetime import timedelta
from types import NoneType
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
import numpy as np
import regex as re
@@ -64,6 +64,7 @@ from vllm.profiler.wrapper import (
CudaProfilerWrapper,
ProtonProfilerWrapper,
TorchProfilerWrapper,
create_graph_capture_profiler,
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
@@ -142,6 +143,7 @@ def maybe_rocm_profiling_fallback(profile_result: MemoryProfilingResult) -> int
if TYPE_CHECKING:
from vllm.device_allocator.sleep_mode_backend import SleepModeBackend
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as GPUModelRunnerV2
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -810,7 +812,8 @@ class Worker(WorkerBase):
cuda_graph_memory_bytes = 0
if not self.model_config.enforce_eager:
cuda_graph_memory_bytes = self.model_runner.capture_model()
with self._get_cudagraph_capture_context():
cuda_graph_memory_bytes = self.model_runner.capture_model()
# Compare actual vs estimated CUDA graph memory (if we did profiling)
if (
@@ -949,6 +952,21 @@ class Worker(WorkerBase):
encoder=self.compilation_config.encoder_compilation_time,
)
def _get_cudagraph_capture_context(self) -> AbstractContextManager[None]:
"""Let the configured profiler observe CUDA graph capture."""
if not self.use_v2_model_runner:
return nullcontext()
if self.profiler is None:
model_runner = cast("GPUModelRunnerV2", self.model_runner)
if not model_runner.needs_cudagraph_capture():
return nullcontext()
self.profiler = create_graph_capture_profiler(
self.profiler_config, global_rank=self.rank
)
if self.profiler is None:
return nullcontext()
return self.profiler.capture_cuda_graphs()
def reset_mm_cache(self) -> None:
self.model_runner.reset_mm_cache()
@@ -1248,6 +1266,9 @@ class Worker(WorkerBase):
else:
trace_name = rank_suffix
if profiler_type == "proton" and self.profiler is not None:
self.profiler.set_output_name(trace_name)
# Create the profiler wrapper only on the first start call
if self.profiler is None:
if profiler_type == "torch":
@@ -1284,7 +1305,9 @@ class Worker(WorkerBase):
try:
self.profiler.stop()
finally:
if self.profiler_config.profiler == "proton":
if self.profiler_config.profiler == "proton" and not (
self.profiler.has_cuda_graph_session
):
# Proton output names are fixed when the wrapper is constructed.
# Recreate it so the next profile_prefix is honored.
self.profiler = None
@@ -70,6 +70,9 @@ class MMEncoderModelRunner(GPUModelRunner):
def capture_model(self, *, profile_only: bool = False) -> int:
return 0
def needs_cudagraph_capture(self) -> bool:
return False
def _dummy_run(
self, *args: Any, **kwargs: Any
) -> tuple[torch.Tensor, torch.Tensor]: