diff --git a/docs/contributing/profiling.md b/docs/contributing/profiling.md index e420a3aa68..880d065a2c 100644 --- a/docs/contributing/profiling.md +++ b/docs/contributing/profiling.md @@ -90,18 +90,18 @@ $ curl -X POST http://localhost:8000/stop_profile is Triton's GPU profiler. It can collect a low-overhead aggregate tree or a Chrome trace and works through the same vLLM profiling controls as the PyTorch and CUDA profilers. Proton currently supports NVIDIA GPUs through CUPTI and -requires eager execution. +supports CUDA graph attribution. -Start a server with a local output directory: +Start a server with a local output directory and graph attribution: ```bash vllm serve meta-llama/Llama-3.1-8B-Instruct \ - --enforce-eager \ --profiler-config '{ "profiler": "proton", "proton_profiler_dir": "./proton_profile", "proton_output_format": "hatchet", - "proton_hook": "triton" + "proton_hook": "triton", + "proton_graph_attribution": true }' ``` @@ -111,7 +111,7 @@ rank-qualified output name, such as `proton_dp0_pp0_tp0_dcp0_ep0_rank0_pid1234_0123456789abcdef0123456789abcdef_run0.hatchet`, so distributed workers, restarted servers, and repeated profiling runs do not overwrite one another. A `profile_prefix` is included when supplied. Each -profile is finalized by `/stop_profile` and is ready to inspect immediately. +profile is written by `/stop_profile` and is ready to inspect immediately. The Proton-specific options are: @@ -121,21 +121,39 @@ The Proton-specific options are: - `proton_mode`: an optional backend mode string - `proton_hook`: `triton` to record Triton launch metadata, or unset - `proton_output_format`: `hatchet`, `hatchet_msgpack`, `chrome_trace`, or unset +- `proton_graph_attribution`: observe CUDA graph capture for replay attribution; + disabled by default and requires `proton_data: "tree"` `hatchet` and `hatchet_msgpack` require `proton_data: "tree"`, while -`chrome_trace` requires `proton_data: "trace"`. CUDA graph profiling is not yet -supported, so `--enforce-eager` is required for every Proton mode. +`chrome_trace` requires `proton_data: "trace"`. Automatic backend selection is recommended. vLLM currently supports Proton's `cupti` backend on NVIDIA GPUs. ROCm support is not yet available. vLLM does not expose Proton's experimental instrumentation backend because current upstream -Triton builds can produce profiles without timing metrics. Backend-specific -modes, including `pcsampling`, can be selected with `proton_mode`. +can produce profiles without timing metrics. When `proton_graph_attribution` is +enabled, Proton observes vLLM's CUDA graph capture with the configured profiling +session active, then deactivates that same session until profiling starts. This +lets later profiles attribute replayed kernels without retaining model-startup +activity. Backend-specific modes can be selected with `proton_mode`; +`pcsampling` synchronizes the CUDA context and therefore requires +`--enforce-eager`. When CUDA graphs are enabled (including encoder graphs), +Proton requires `proton_graph_attribution: true` to collect replayed kernels. +For Chrome traces, disable CUDA graphs with `--enforce-eager`. -Triton 3.6 supports explicit `hatchet` and `chrome_trace` output. The -`hatchet_msgpack` format and `periodic_flushing` mode require Triton 3.7 or -newer; vLLM rejects these options with the detected version before starting a -session. +CUDA graph-attributed profiles support repeated `start_profile`/`stop_profile` +runs. Each stop flushes and writes one tree-data phase while preserving the +graph-aware session. Without `proton_graph_attribution`, each `stop_profile` +instead finalizes and writes an independent Proton session. + +CUDA graph attribution requires Triton 3.7 or newer. It uses the phase data API +to discard graph-capture activity and separate profiling runs. The `hatchet_msgpack` +output format and `periodic_flushing` mode also require Triton 3.7 or newer. +`periodic_flushing` cannot be combined with graph attribution because both +manage the session's data phases. Ordinary Proton profiling remains available +with Triton 3.6. + +Graph attribution retains capture metadata for the worker lifetime. With eager +execution or no graphs to capture, profiling uses ordinary independent sessions. Inspect tree profiles with: diff --git a/tests/v1/worker/test_gpu_profiler.py b/tests/v1/worker/test_gpu_profiler.py index 347b5b379a..f6263b68da 100644 --- a/tests/v1/worker/test_gpu_profiler.py +++ b/tests/v1/worker/test_gpu_profiler.py @@ -350,17 +350,24 @@ def test_profiler_entered_during_capture(): def make_proton(session_id: int | None = 7): + data = SimpleNamespace( + advance_phase=Mock(side_effect=range(1, 100)), + clear=Mock(), + get=Mock(return_value={"traceEvents": []}), + get_msgpack=Mock(return_value=b"profile"), + ) return SimpleNamespace( start=Mock(return_value=session_id), activate=Mock(), deactivate=Mock(), finalize=Mock(), scope=Mock(return_value=nullcontext()), + data=data, ) def make_proton_wrapper( - tmp_path, proton=None, triton_version="3.6.0", **config_overrides + tmp_path, proton=None, triton_version="3.7.0", **config_overrides ): proton = proton or make_proton() config = ProfilerConfig( @@ -415,6 +422,13 @@ class TestProtonConfig: }, "requires proton_data", ), + ( + { + "proton_data": "trace", + "proton_graph_attribution": True, + }, + "requires proton_data='tree'", + ), ], ) def test_rejects_invalid_option_combinations(self, tmp_path, options, message): @@ -450,6 +464,7 @@ class TestProtonConfig: ("proton_mode", "pcsampling"), ("proton_hook", "triton"), ("proton_output_format", "chrome_trace"), + ("proton_graph_attribution", True), ], ) def test_rejects_proton_options_for_other_profilers(self, field, value): @@ -478,15 +493,101 @@ class TestProtonConfig: compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.NONE), ) - def test_rejects_proton_when_cuda_graphs_are_enabled(self, tmp_path): - with pytest.raises(ValueError, match="requires CUDA graphs to be disabled"): + @pytest.mark.parametrize("encoder_only", [False, True]) + @pytest.mark.parametrize("attribution", [False, True]) + def test_cuda_graphs_require_attribution(self, tmp_path, encoder_only, attribution): + # Encoder graphs are independent of the decoder's cudagraph_mode. + expected = ( + nullcontext() + if attribution + else pytest.raises( + ValueError, match="requires proton_graph_attribution=True" + ) + ) + with expected: + VllmConfig( + profiler_config=ProfilerConfig( + profiler="proton", + proton_profiler_dir=str(tmp_path), + proton_graph_attribution=attribution, + ), + compilation_config=CompilationConfig( + cudagraph_mode=( + CUDAGraphMode.NONE if encoder_only else CUDAGraphMode.FULL + ), + cudagraph_mm_encoder=encoder_only, + ), + ) + + def test_validates_default_cuda_graph_mode_after_resolution(self, tmp_path): + with pytest.raises(ValueError, match="requires proton_graph_attribution=True"): VllmConfig( profiler_config=ProfilerConfig( profiler="proton", proton_profiler_dir=str(tmp_path) ), - compilation_config=CompilationConfig( - cudagraph_mode=CUDAGraphMode.PIECEWISE + ) + + @pytest.mark.parametrize( + "mode", + [ + "periodic_flushing", + "periodic_flushing:format=hatchet", + "PERIODIC_FLUSHING:format=hatchet", + ], + ) + def test_graph_attribution_rejects_periodic_flushing(self, tmp_path, mode): + # Reject before Proton's native phase manager can abort the worker. + with pytest.raises(ValueError, match="incompatible with periodic_flushing"): + ProfilerConfig( + profiler="proton", + proton_profiler_dir=str(tmp_path), + proton_graph_attribution=True, + proton_mode=mode, + ) + + @pytest.mark.parametrize("attribution", [False, True]) + @pytest.mark.parametrize( + "mode", ["pcsampling", "pcsampling:interval=100", "PcSampling:interval=100"] + ) + @pytest.mark.parametrize("encoder_only", [False, True]) + def test_pcsampling_requires_graphs_disabled( + self, tmp_path, attribution, mode, encoder_only + ): + with pytest.raises(ValueError, match="PC sampling requires CUDA graphs"): + VllmConfig( + profiler_config=ProfilerConfig( + profiler="proton", + proton_profiler_dir=str(tmp_path), + proton_graph_attribution=attribution, + proton_mode=mode, ), + compilation_config=CompilationConfig( + cudagraph_mode=CUDAGraphMode.NONE + if encoder_only + else CUDAGraphMode.FULL, + cudagraph_mm_encoder=encoder_only, + ), + ) + + def test_ordinary_proton_keeps_mrv1_graph_restriction(self, tmp_path, monkeypatch): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") + with pytest.raises(ValueError, match="requires proton_graph_attribution=True"): + VllmConfig( + profiler_config=ProfilerConfig( + profiler="proton", proton_profiler_dir=str(tmp_path) + ), + compilation_config=CompilationConfig(cudagraph_mode=CUDAGraphMode.FULL), + ) + + def test_graph_attribution_rejects_mrv1(self, tmp_path, monkeypatch): + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") + with pytest.raises(ValueError, match="requires the V2 model runner"): + VllmConfig( + profiler_config=ProfilerConfig( + profiler="proton", + proton_profiler_dir=str(tmp_path), + proton_graph_attribution=True, + ) ) @@ -564,7 +665,20 @@ class TestProtonProfilerWrapper: ) def test_newer_features_reject_triton_3_6(self, tmp_path, option, value, feature): with pytest.raises(RuntimeError, match=feature): - make_proton_wrapper(tmp_path, **{option: value}) + make_proton_wrapper(tmp_path, triton_version="3.6.0", **{option: value}) + + @pytest.mark.parametrize("version", ["3.6.0", "unknown"]) + def test_graph_attribution_requires_phase_api(self, tmp_path, version): + with pytest.raises(RuntimeError, match="requires Triton >= 3.7"): + make_proton_wrapper( + tmp_path, triton_version=version, proton_graph_attribution=True + ) + + def test_ordinary_profiling_still_supports_triton_3_6(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, triton_version="3.6.0") + wrapper.start() + wrapper.stop() + proton.finalize.assert_called_once_with(session=7) @pytest.mark.parametrize( ("option", "value"), @@ -583,6 +697,125 @@ class TestProtonProfilerWrapper: with pytest.raises(RuntimeError, match="does not support selecting"): make_proton_wrapper(tmp_path, proton, proton_output_format="hatchet") + def test_cuda_graph_tree_phase_is_written_at_stop(self, tmp_path): + wrapper, proton = make_proton_wrapper( + tmp_path, + proton_context="python", + proton_graph_attribution=True, + ) + with wrapper.capture_cuda_graphs(): + proton.start.assert_called_once() + proton.deactivate.assert_not_called() + + capture_args = proton.start.call_args.kwargs + assert capture_args["context"] == "python" + assert capture_args["data"] == "tree" + proton.data.advance_phase.assert_called_once_with(7) + proton.deactivate.assert_called_once_with(session=7, flushing=True) + proton.data.clear.assert_called_once_with(7, 0) + + wrapper.start() + wrapper.stop() + wrapper.start() + wrapper.stop() + + assert proton.data.get.call_args_list == [call(7, 1), call(7, 2)] + proton.data.clear.assert_has_calls([call(7, 0), call(7, 1), call(7, 2)]) + output_names = sorted(tmp_path.glob("proton_rank_3_*.hatchet")) + assert len(output_names) == 2 + assert output_names[0].name.endswith("_run0.hatchet") + assert output_names[1].name.endswith("_run1.hatchet") + + def test_capture_is_noop_without_opt_in(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path) + with wrapper.capture_cuda_graphs(): + pass + proton.start.assert_not_called() + + @pytest.mark.parametrize("delay", [0, 2]) + def test_duplicate_start_preserves_output_prefix(self, tmp_path, delay): + wrapper, proton = make_proton_wrapper( + tmp_path, proton_graph_attribution=True, delay_iterations=delay + ) + with wrapper.capture_cuda_graphs(): + pass + wrapper.set_output_name("first") + wrapper.start() + wrapper.set_output_name("duplicate") + wrapper.start() + for _ in range(delay): + wrapper.step() + wrapper.stop() + assert len(list(tmp_path.glob("proton_first_*.hatchet"))) == 1 + assert not list(tmp_path.glob("proton_duplicate_*")) + wrapper.set_output_name("second") + wrapper.start() + for _ in range(delay): + wrapper.step() + wrapper.stop() + assert len(list(tmp_path.glob("proton_second_*.hatchet"))) == 1 + + def test_failed_export_clears_activity_and_allows_next_interval(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True) + with wrapper.capture_cuda_graphs(): + pass + proton.data.get.side_effect = [OSError("disk full"), []] + wrapper.start() + wrapper.stop() + proton.data.clear.assert_called_with(7, 1) + wrapper.start() + wrapper.stop() + proton.data.clear.assert_called_with(7, 2) + assert len(list(tmp_path.glob("*_run1.hatchet"))) == 1 + + def test_cuda_graph_context_deactivates_after_capture_error(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True) + + with ( + pytest.raises(RuntimeError, match="capture failed"), + wrapper.capture_cuda_graphs(), + ): + raise RuntimeError("capture failed") + + proton.deactivate.assert_called_once_with(session=7, flushing=True) + + def test_cuda_graph_capture_deactivates_when_phase_advance_fails(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True) + proton.data.advance_phase.side_effect = RuntimeError("advance failed") + + with ( + pytest.raises(RuntimeError, match="advance failed"), + wrapper.capture_cuda_graphs(), + ): + pass + + proton.deactivate.assert_called_once_with(session=7, flushing=True) + proton.data.clear.assert_not_called() + + def test_cuda_graph_stop_deactivates_when_phase_advance_fails(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True) + with wrapper.capture_cuda_graphs(): + pass + proton.data.clear.reset_mock() + proton.data.advance_phase.side_effect = RuntimeError("advance failed") + + wrapper._start() + with pytest.raises(RuntimeError, match="advance failed"): + wrapper._stop() + + proton.deactivate.assert_called_with(session=7, flushing=True) + proton.data.clear.assert_not_called() + + def test_shutdown_finalizes_cuda_graph_capture_session(self, tmp_path): + wrapper, proton = make_proton_wrapper(tmp_path, proton_graph_attribution=True) + with wrapper.capture_cuda_graphs(): + pass + + wrapper.shutdown() + wrapper.shutdown() + + proton.finalize.assert_called_once_with(session=7) + def test_missing_proton_has_actionable_error(self, tmp_path): config = ProfilerConfig(profiler="proton", proton_profiler_dir=str(tmp_path)) with ( @@ -638,6 +871,7 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run(): ), patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper, ): + wrapper.return_value.has_cuda_graph_session = False Worker.profile(worker, profile_prefix="first") Worker.profile(worker, is_start=False) Worker.profile(worker, profile_prefix="second") @@ -647,3 +881,157 @@ def test_gpu_worker_recreates_proton_profiler_for_each_run(): call(worker.profiler_config, worker_name="second_rank1"), ] assert wrapper.return_value.start.call_count == 2 + + +@_requires_cuda_for_proton +def test_gpu_worker_reuses_cuda_graph_proton_session(): + worker = MagicMock() + worker.rank = 1 + worker.profiler = MagicMock(spec=ProtonProfilerWrapper) + worker.profiler.has_cuda_graph_session = True + worker.profiler_config.profiler = "proton" + + with patch( + "vllm.distributed.utils.get_worker_rank_suffix", + return_value="rank1", + ): + Worker.profile(worker, profile_prefix="first") + Worker.profile(worker, is_start=False) + + worker.profiler.set_output_name.assert_called_once_with("first_rank1") + worker.profiler.start.assert_called_once_with() + worker.profiler.stop.assert_called_once_with() + + +@_requires_cuda_for_proton +@pytest.mark.parametrize("runner", ["attribution_off", "v1", "no_capture"]) +def test_proton_not_initialized_without_capture(runner): + worker = MagicMock() + worker.profiler = None + worker.profiler_config.profiler = "proton" + worker.profiler_config.proton_graph_attribution = runner != "attribution_off" + worker.use_v2_model_runner = runner != "v1" + worker.model_runner.needs_cudagraph_capture.return_value = runner != "no_capture" + + context = Worker._get_cudagraph_capture_context(worker) + + assert worker.profiler is None + with context: + pass + + +@_requires_cuda_for_proton +def test_proton_initializes_before_cuda_graph_capture(): + class FakeProtonProfiler: + def __init__(self, config, worker_name): + self.config = config + self.worker_name = worker_name + self.capture_context = nullcontext() + + def capture_cuda_graphs(self): + return self.capture_context + + worker = MagicMock() + worker.rank = 2 + worker.profiler = None + worker.profiler_config.profiler = "proton" + worker.profiler_config.proton_graph_attribution = True + worker.use_v2_model_runner = True + worker.model_runner.needs_cudagraph_capture.return_value = True + + with ( + patch( + "vllm.distributed.utils.get_worker_rank_suffix", + return_value="rank2", + ), + patch( + "vllm.profiler.wrapper.ProtonProfilerWrapper", + FakeProtonProfiler, + ), + ): + context = Worker._get_cudagraph_capture_context(worker) + + assert worker.profiler.config is worker.profiler_config + assert worker.profiler.worker_name == "rank2" + assert context is worker.profiler.capture_context + + +@_requires_cuda_for_proton +@pytest.mark.parametrize("context", ["shadow", "python"]) +@pytest.mark.parametrize("output_format", ["hatchet", "hatchet_msgpack"]) +def test_proton_cuda_graph_replay_attribution_on_gpu(tmp_path, context, output_format): + """Both intervals contain replay kernels, without capture-only activity.""" + import json + + import torch + import triton + import triton.profiler as proton + from packaging.version import Version + + if Version(triton.__version__) < Version("3.7"): + pytest.skip("Graph attribution requires Triton >= 3.7") + + wrapper = ProtonProfilerWrapper( + ProfilerConfig( + profiler="proton", + proton_profiler_dir=str(tmp_path), + proton_graph_attribution=True, + proton_context=context, + proton_output_format=output_format, + ), + worker_name="gpu", + ) + worker = SimpleNamespace( + profiler=wrapper, + use_v2_model_runner=True, + ) + x = torch.ones(1024, device="cuda") + graph = torch.cuda.CUDAGraph() + + def capture_only(): + x.add_(1) + + def captured_add(): + x.add_(1) + + def kernel_metrics(value): + if isinstance(value, list): + return [metric for child in value for metric in kernel_metrics(child)] + if isinstance(value, dict): + metrics = value.get("metrics", {}) + return ([metrics] if metrics.get("count", 0) else []) + [ + metric for child in value.values() for metric in kernel_metrics(child) + ] + return [] + + try: + with Worker._get_cudagraph_capture_context(worker): + with proton.scope("capture_only"): + capture_only() + torch.accelerator.synchronize() + with torch.cuda.graph(graph), proton.scope("captured_add"): + captured_add() + for run in range(2): + wrapper.start() + with wrapper.annotate_context_manager(f"replay_{run}"): + graph.replay() + wrapper.stop() + (path,) = tmp_path.glob(f"*_run{run}.{output_format}") + if output_format == "hatchet_msgpack": + import msgpack + + data = msgpack.unpackb(path.read_bytes()) + else: + data = json.loads(path.read_text()) + serialized = json.dumps(data) + assert "" in serialized + assert "captured_add" in serialized + assert "capture_only" not in serialized + assert f"replay_{1 - run}" not in serialized + metrics = kernel_metrics(data) + assert sum(metric["count"] for metric in metrics) == 1 + assert sum(metric["time (ns)"] for metric in metrics) > 0 + torch.testing.assert_close(x, torch.full_like(x, 4)) + finally: + wrapper.shutdown() + assert not list(tmp_path.glob(".proton_cuda_graph_session*")) diff --git a/vllm/config/profiler.py b/vllm/config/profiler.py index b3b232490f..535960a8d7 100644 --- a/vllm/config/profiler.py +++ b/vllm/config/profiler.py @@ -76,6 +76,10 @@ class ProfilerConfig: """Optional format passed to Proton when finalizing a profile. ``None`` uses the default format for ``proton_data``.""" + proton_graph_attribution: bool = False + """Observe CUDA graph capture so replayed kernels can be attributed. + Requires Triton >= 3.7 and ``proton_data='tree'``.""" + torch_profiler_with_stack: bool = True """If `True`, enables stack tracing in the torch profiler. Enabled by default as it is useful for debugging. Can be disabled via @@ -199,6 +203,7 @@ class ProfilerConfig: ("proton_mode", self.proton_mode, None), ("proton_hook", self.proton_hook, None), ("proton_output_format", self.proton_output_format, None), + ("proton_graph_attribution", self.proton_graph_attribution, False), ) if value != default ] @@ -220,6 +225,17 @@ class ProfilerConfig: if self.profiler == "proton": output_format = self.proton_output_format + if self.proton_graph_attribution and self.proton_data != "tree": + raise ValueError("proton_graph_attribution requires proton_data='tree'") + if ( + self.proton_graph_attribution + and self.proton_mode + and self.proton_mode.split(":", 1)[0].lower() == "periodic_flushing" + ): + raise ValueError( + "proton_graph_attribution is incompatible with periodic_flushing: " + "both manage Proton data phases." + ) if output_format == "chrome_trace" and self.proton_data != "trace": raise ValueError("chrome_trace output requires proton_data='trace'") if ( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2fcc144af0..7d2baea26c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1551,18 +1551,6 @@ class VllmConfig: self.compilation_config.mode = CompilationMode.NONE self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE - if self.profiler_config.profiler == "proton": - if not current_platform.is_cuda(): - raise ValueError( - "The Proton profiler currently supports NVIDIA CUDA only" - ) - if self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE: - raise ValueError( - "The Proton profiler requires CUDA graphs to be disabled. " - "Use --enforce-eager or set " - "--compilation-config.cudagraph_mode=none." - ) - if os.environ.get("TORCH_COMPILE_DISABLE") == "1": logger.warning_once( "TORCH_COMPILE_DISABLE is set, disabling torch.compile. " @@ -1860,6 +1848,7 @@ class VllmConfig: else: self._validate_v1_model_runner() + self._validate_profiler_config() self._validate_batch_sharded_sampling() self._validate_adaptive_verification() @@ -3042,6 +3031,44 @@ class VllmConfig: return unsupported + def _validate_profiler_config(self) -> None: + if self.profiler_config.profiler != "proton": + return + + from vllm.platforms import current_platform + + if not current_platform.is_cuda(): + raise ValueError("The Proton profiler currently supports NVIDIA CUDA only") + if ( + self.profiler_config.proton_graph_attribution + and not self.use_v2_model_runner + ): + raise ValueError( + "Proton CUDA graph attribution requires the V2 model runner." + ) + + has_cuda_graphs = ( + self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE + or ( + self.compilation_config.cudagraph_mm_encoder + and not (self.model_config and self.model_config.enforce_eager) + ) + ) + if not has_cuda_graphs: + return + mode = self.profiler_config.proton_mode + if mode and mode.split(":", 1)[0].lower() == "pcsampling": + raise ValueError( + "Proton PC sampling requires CUDA graphs to be disabled. " + "Use --enforce-eager." + ) + if not self.profiler_config.proton_graph_attribution: + raise ValueError( + "Proton profiling with CUDA graphs requires " + "proton_graph_attribution=True to capture replayed kernels. " + "Enable attribution or use --enforce-eager." + ) + def _validate_v2_model_runner(self) -> None: """Check for features not yet supported by the V2 model runner.""" if not HAS_TRITON: diff --git a/vllm/profiler/wrapper.py b/vllm/profiler/wrapper.py index fb89631986..abe8a34317 100644 --- a/vllm/profiler/wrapper.py +++ b/vllm/profiler/wrapper.py @@ -6,8 +6,9 @@ import inspect import json import os from abc import ABC, abstractmethod -from collections.abc import Callable -from contextlib import nullcontext +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress +from glob import glob from typing import Literal from uuid import uuid4 @@ -160,6 +161,15 @@ class WorkerProfiler(ABC): if self._running: self.stop() + @property + def has_cuda_graph_session(self) -> bool: + """Whether a capture-time session is retained to attribute replays.""" + return False + + def capture_cuda_graphs(self) -> AbstractContextManager[None]: + """Observe graph creation for backends that attribute replay activity.""" + return nullcontext() + def annotate_context_manager(self, name: str): """Return a context manager to annotate profiler traces.""" return nullcontext() @@ -387,6 +397,7 @@ class ProtonProfilerWrapper(WorkerProfiler): self._mode = profiler_config.proton_mode self._hook = profiler_config.proton_hook self._output_format = profiler_config.proton_output_format + self._graph_attribution = profiler_config.proton_graph_attribution self._triton_version_string = getattr(triton, "__version__", "unknown") try: self._triton_version = Version(self._triton_version_string) @@ -398,6 +409,13 @@ class ProtonProfilerWrapper(WorkerProfiler): # worker cannot overwrite profiles left by an earlier server process. self._instance_id = f"pid{os.getpid()}_{uuid4().hex}" self._run_id = 0 + self._graph_session = False + self._phase = 0 + self._active_output_path: str | None = None + self._session_storage_path = os.path.join( + self._output_dir, + f".proton_cuda_graph_session_{worker_name}_{self._instance_id}", + ) logger.info_once( "Proton profiling enabled. Output will be saved under: %s", @@ -412,6 +430,10 @@ class ProtonProfilerWrapper(WorkerProfiler): ) def _validate_capabilities(self) -> None: + if self._graph_attribution: + self._require_triton_version( + "CUDA graph attribution", _TRITON_PROTON_3_7_VERSION + ) if self._output_format is not None: parameters = inspect.signature(self._proton.finalize).parameters supports_output_format = "output_format" in parameters or any( @@ -428,7 +450,7 @@ class ProtonProfilerWrapper(WorkerProfiler): self._require_triton_version( "hatchet_msgpack output", _TRITON_PROTON_3_7_VERSION ) - if self._mode and self._mode.split(":", 1)[0] == "periodic_flushing": + if self._mode and self._mode.split(":", 1)[0].lower() == "periodic_flushing": self._require_triton_version( "periodic flushing", _TRITON_PROTON_3_7_VERSION ) @@ -447,28 +469,112 @@ class ProtonProfilerWrapper(WorkerProfiler): raise RuntimeError("Proton did not create a profiling session") return session_id + @property + def has_cuda_graph_session(self) -> bool: + return self._graph_session + + def set_output_name(self, worker_name: str) -> None: + """Set the next run's output name after startup graph capture.""" + if self._active: + return + self._output_path = os.path.join(self._output_dir, f"proton_{worker_name}") + + @override + @contextmanager + def capture_cuda_graphs(self) -> Iterator[None]: + """Keep a Proton session active while vLLM captures CUDA graphs.""" + if not self._graph_attribution: + yield + return + if self._session_id is None: + self._session_id = self._create_session(self._session_storage_path) + else: + self._proton.activate(session=self._session_id) + self._graph_session = True + + try: + yield + finally: + captured_phase = self._phase + try: + self._phase = self._proton.data.advance_phase(self._session_id) + finally: + self._proton.deactivate(session=self._session_id, flushing=True) + self._proton.data.clear(self._session_id, captured_phase) + @override def _start(self) -> None: - output_path = f"{self._output_path}_{self._instance_id}_run{self._run_id}" - self._session_id = self._create_session(output_path) + self._active_output_path = ( + f"{self._output_path}_{self._instance_id}_run{self._run_id}" + ) self._run_id += 1 + if self._graph_session: + assert self._session_id is not None + self._proton.activate(session=self._session_id) + else: + self._session_id = self._create_session(self._active_output_path) + + def _write_graph_phase(self, phase: int) -> None: + assert self._active_output_path is not None + output_format = self._output_format or "hatchet" + output_path = f"{self._active_output_path}.{output_format}" + if output_format == "hatchet_msgpack": + with open(output_path, "wb") as output_file: + output_file.write( + self._proton.data.get_msgpack(self._session_id, phase) + ) + else: + with open(output_path, "w", encoding="utf-8") as output_file: + json.dump(self._proton.data.get(self._session_id, phase), output_file) + + def _finalize_session(self, session_id: int) -> None: + if self._output_format is None: + self._proton.finalize(session=session_id) + else: + self._proton.finalize(session=session_id, output_format=self._output_format) @override def _stop(self) -> None: assert self._session_id is not None session_id = self._session_id + if self._graph_session: + completed_phase = self._phase + try: + self._phase = self._proton.data.advance_phase(session_id) + finally: + self._proton.deactivate(session=session_id, flushing=True) + try: + self._write_graph_phase(completed_phase) + finally: + self._proton.data.clear(session_id, completed_phase) + self._active_output_path = None + return + try: self._proton.deactivate(session=session_id) finally: try: - if self._output_format is None: - self._proton.finalize(session=session_id) - else: - self._proton.finalize( - session=session_id, output_format=self._output_format - ) + self._finalize_session(session_id) finally: self._session_id = None + self._active_output_path = None + + @override + def shutdown(self) -> None: + super().shutdown() + if self._graph_session and self._session_id is not None: + session_id = self._session_id + self._session_id = None + try: + self._finalize_session(session_id) + except Exception: + logger.exception( + "Failed to finalize Proton CUDA graph session during shutdown." + ) + finally: + for output_path in glob(f"{self._session_storage_path}.*"): + with suppress(FileNotFoundError): + os.remove(output_path) @override def annotate_context_manager(self, name: str): @@ -496,3 +602,24 @@ class CudaProfilerWrapper(WorkerProfiler): @override def annotate_context_manager(self, name: str): return torch.cuda.nvtx.range(name) + + +def create_graph_capture_profiler( + profiler_config: ProfilerConfig, global_rank: int +) -> WorkerProfiler | None: + """Create a profiler to observe CUDA graph capture, if configured. + + Applies only to backends that attribute graph replay activity which + need a session around capture (Proton with ``proton_graph_attribution``). + """ + if ( + profiler_config.profiler == "proton" + and profiler_config.proton_graph_attribution + ): + from vllm.distributed.utils import get_worker_rank_suffix + + return ProtonProfilerWrapper( + profiler_config, + worker_name=get_worker_rank_suffix(global_rank=global_rank), + ) + return None diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index d66fc6c90f..4b1520f276 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -987,6 +987,16 @@ class GPUModelRunner(LoRAModelRunnerMixin): """Estimate the GPU memory required to capture CUDA graphs.""" return _profile_cudagraph_memory(self) + def needs_cudagraph_capture(self) -> bool: + """Whether capture_model() has any CUDA graphs left to capture.""" + return ( + self.model_state.supports_mm_inputs + and self.model_state.encoder_runner.has_cudagraph() + ) or ( + self.cudagraph_manager is not None + and self.cudagraph_manager.needs_capture() + ) + @torch.inference_mode() def capture_model(self, *, profile_only: bool = False) -> int: assert self.cudagraph_manager is not None diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 2e84fd511a..3f457b9b7b 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -9,7 +9,7 @@ from collections.abc import Callable from contextlib import AbstractContextManager, contextmanager, nullcontext from datetime import timedelta from types import NoneType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast import numpy as np import regex as re @@ -64,6 +64,7 @@ from vllm.profiler.wrapper import ( CudaProfilerWrapper, ProtonProfilerWrapper, TorchProfilerWrapper, + create_graph_capture_profiler, ) from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask @@ -142,6 +143,7 @@ def maybe_rocm_profiling_fallback(profile_result: MemoryProfilingResult) -> int if TYPE_CHECKING: from vllm.device_allocator.sleep_mode_backend import SleepModeBackend from vllm.model_executor.model_loader.tensorizer import TensorizerConfig + from vllm.v1.worker.gpu.model_runner import GPUModelRunner as GPUModelRunnerV2 from vllm.v1.worker.gpu_model_runner import GPUModelRunner @@ -810,7 +812,8 @@ class Worker(WorkerBase): cuda_graph_memory_bytes = 0 if not self.model_config.enforce_eager: - cuda_graph_memory_bytes = self.model_runner.capture_model() + with self._get_cudagraph_capture_context(): + cuda_graph_memory_bytes = self.model_runner.capture_model() # Compare actual vs estimated CUDA graph memory (if we did profiling) if ( @@ -949,6 +952,21 @@ class Worker(WorkerBase): encoder=self.compilation_config.encoder_compilation_time, ) + def _get_cudagraph_capture_context(self) -> AbstractContextManager[None]: + """Let the configured profiler observe CUDA graph capture.""" + if not self.use_v2_model_runner: + return nullcontext() + if self.profiler is None: + model_runner = cast("GPUModelRunnerV2", self.model_runner) + if not model_runner.needs_cudagraph_capture(): + return nullcontext() + self.profiler = create_graph_capture_profiler( + self.profiler_config, global_rank=self.rank + ) + if self.profiler is None: + return nullcontext() + return self.profiler.capture_cuda_graphs() + def reset_mm_cache(self) -> None: self.model_runner.reset_mm_cache() @@ -1248,6 +1266,9 @@ class Worker(WorkerBase): else: trace_name = rank_suffix + if profiler_type == "proton" and self.profiler is not None: + self.profiler.set_output_name(trace_name) + # Create the profiler wrapper only on the first start call if self.profiler is None: if profiler_type == "torch": @@ -1284,7 +1305,9 @@ class Worker(WorkerBase): try: self.profiler.stop() finally: - if self.profiler_config.profiler == "proton": + if self.profiler_config.profiler == "proton" and not ( + self.profiler.has_cuda_graph_session + ): # Proton output names are fixed when the wrapper is constructed. # Recreate it so the next profile_prefix is honored. self.profiler = None diff --git a/vllm/v1/worker/mm_encoder_model_runner.py b/vllm/v1/worker/mm_encoder_model_runner.py index 8208a6ce3d..e19f1b332a 100644 --- a/vllm/v1/worker/mm_encoder_model_runner.py +++ b/vllm/v1/worker/mm_encoder_model_runner.py @@ -70,6 +70,9 @@ class MMEncoderModelRunner(GPUModelRunner): def capture_model(self, *, profile_only: bool = False) -> int: return 0 + def needs_cudagraph_capture(self) -> bool: + return False + def _dummy_run( self, *args: Any, **kwargs: Any ) -> tuple[torch.Tensor, torch.Tensor]: