[Profiler] Add Proton CUDA graph attribution for MRV2 (#51084)

Signed-off-by: Luossu <zhluosuu@outlook.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
鐘天楽
2026-09-14 19:02:07 +00:00
committed by GitHub
co-authored by Nick Hill
parent 5372e72a98
commit 0a5747d410
8 changed files with 657 additions and 45 deletions
+138 -11
View File
@@ -6,8 +6,9 @@ import inspect
import json
import os
from abc import ABC, abstractmethod
from collections.abc import Callable
from contextlib import nullcontext
from collections.abc import Callable, Iterator
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
from glob import glob
from typing import Literal
from uuid import uuid4
@@ -160,6 +161,15 @@ class WorkerProfiler(ABC):
if self._running:
self.stop()
@property
def has_cuda_graph_session(self) -> bool:
"""Whether a capture-time session is retained to attribute replays."""
return False
def capture_cuda_graphs(self) -> AbstractContextManager[None]:
"""Observe graph creation for backends that attribute replay activity."""
return nullcontext()
def annotate_context_manager(self, name: str):
"""Return a context manager to annotate profiler traces."""
return nullcontext()
@@ -387,6 +397,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
self._mode = profiler_config.proton_mode
self._hook = profiler_config.proton_hook
self._output_format = profiler_config.proton_output_format
self._graph_attribution = profiler_config.proton_graph_attribution
self._triton_version_string = getattr(triton, "__version__", "unknown")
try:
self._triton_version = Version(self._triton_version_string)
@@ -398,6 +409,13 @@ class ProtonProfilerWrapper(WorkerProfiler):
# worker cannot overwrite profiles left by an earlier server process.
self._instance_id = f"pid{os.getpid()}_{uuid4().hex}"
self._run_id = 0
self._graph_session = False
self._phase = 0
self._active_output_path: str | None = None
self._session_storage_path = os.path.join(
self._output_dir,
f".proton_cuda_graph_session_{worker_name}_{self._instance_id}",
)
logger.info_once(
"Proton profiling enabled. Output will be saved under: %s",
@@ -412,6 +430,10 @@ class ProtonProfilerWrapper(WorkerProfiler):
)
def _validate_capabilities(self) -> None:
if self._graph_attribution:
self._require_triton_version(
"CUDA graph attribution", _TRITON_PROTON_3_7_VERSION
)
if self._output_format is not None:
parameters = inspect.signature(self._proton.finalize).parameters
supports_output_format = "output_format" in parameters or any(
@@ -428,7 +450,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
self._require_triton_version(
"hatchet_msgpack output", _TRITON_PROTON_3_7_VERSION
)
if self._mode and self._mode.split(":", 1)[0] == "periodic_flushing":
if self._mode and self._mode.split(":", 1)[0].lower() == "periodic_flushing":
self._require_triton_version(
"periodic flushing", _TRITON_PROTON_3_7_VERSION
)
@@ -447,28 +469,112 @@ class ProtonProfilerWrapper(WorkerProfiler):
raise RuntimeError("Proton did not create a profiling session")
return session_id
@property
def has_cuda_graph_session(self) -> bool:
return self._graph_session
def set_output_name(self, worker_name: str) -> None:
"""Set the next run's output name after startup graph capture."""
if self._active:
return
self._output_path = os.path.join(self._output_dir, f"proton_{worker_name}")
@override
@contextmanager
def capture_cuda_graphs(self) -> Iterator[None]:
"""Keep a Proton session active while vLLM captures CUDA graphs."""
if not self._graph_attribution:
yield
return
if self._session_id is None:
self._session_id = self._create_session(self._session_storage_path)
else:
self._proton.activate(session=self._session_id)
self._graph_session = True
try:
yield
finally:
captured_phase = self._phase
try:
self._phase = self._proton.data.advance_phase(self._session_id)
finally:
self._proton.deactivate(session=self._session_id, flushing=True)
self._proton.data.clear(self._session_id, captured_phase)
@override
def _start(self) -> None:
output_path = f"{self._output_path}_{self._instance_id}_run{self._run_id}"
self._session_id = self._create_session(output_path)
self._active_output_path = (
f"{self._output_path}_{self._instance_id}_run{self._run_id}"
)
self._run_id += 1
if self._graph_session:
assert self._session_id is not None
self._proton.activate(session=self._session_id)
else:
self._session_id = self._create_session(self._active_output_path)
def _write_graph_phase(self, phase: int) -> None:
assert self._active_output_path is not None
output_format = self._output_format or "hatchet"
output_path = f"{self._active_output_path}.{output_format}"
if output_format == "hatchet_msgpack":
with open(output_path, "wb") as output_file:
output_file.write(
self._proton.data.get_msgpack(self._session_id, phase)
)
else:
with open(output_path, "w", encoding="utf-8") as output_file:
json.dump(self._proton.data.get(self._session_id, phase), output_file)
def _finalize_session(self, session_id: int) -> None:
if self._output_format is None:
self._proton.finalize(session=session_id)
else:
self._proton.finalize(session=session_id, output_format=self._output_format)
@override
def _stop(self) -> None:
assert self._session_id is not None
session_id = self._session_id
if self._graph_session:
completed_phase = self._phase
try:
self._phase = self._proton.data.advance_phase(session_id)
finally:
self._proton.deactivate(session=session_id, flushing=True)
try:
self._write_graph_phase(completed_phase)
finally:
self._proton.data.clear(session_id, completed_phase)
self._active_output_path = None
return
try:
self._proton.deactivate(session=session_id)
finally:
try:
if self._output_format is None:
self._proton.finalize(session=session_id)
else:
self._proton.finalize(
session=session_id, output_format=self._output_format
)
self._finalize_session(session_id)
finally:
self._session_id = None
self._active_output_path = None
@override
def shutdown(self) -> None:
super().shutdown()
if self._graph_session and self._session_id is not None:
session_id = self._session_id
self._session_id = None
try:
self._finalize_session(session_id)
except Exception:
logger.exception(
"Failed to finalize Proton CUDA graph session during shutdown."
)
finally:
for output_path in glob(f"{self._session_storage_path}.*"):
with suppress(FileNotFoundError):
os.remove(output_path)
@override
def annotate_context_manager(self, name: str):
@@ -496,3 +602,24 @@ class CudaProfilerWrapper(WorkerProfiler):
@override
def annotate_context_manager(self, name: str):
return torch.cuda.nvtx.range(name)
def create_graph_capture_profiler(
profiler_config: ProfilerConfig, global_rank: int
) -> WorkerProfiler | None:
"""Create a profiler to observe CUDA graph capture, if configured.
Applies only to backends that attribute graph replay activity which
need a session around capture (Proton with ``proton_graph_attribution``).
"""
if (
profiler_config.profiler == "proton"
and profiler_config.proton_graph_attribution
):
from vllm.distributed.utils import get_worker_rank_suffix
return ProtonProfilerWrapper(
profiler_config,
worker_name=get_worker_rank_suffix(global_rank=global_rank),
)
return None