[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)

Signed-off-by: czhu <conway.zhu@cohere.com>
Signed-off-by: Conway Zhu <conway.zhu@cohere.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
czhu-cohere
2026-10-06 05:21:19 +00:00
committed by GitHub
co-authored by aoshen02
parent 9a1f6fd4cb
commit 6fddc9b21e
25 changed files with 1497 additions and 164 deletions
+104 -26
View File
@@ -5,6 +5,7 @@ import importlib
import inspect
import json
import os
import threading
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator, Sequence
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
@@ -17,7 +18,13 @@ from typing_extensions import override
import vllm.version
from vllm.config import ProfilerConfig
from vllm.config.profiler import ProfilerKind, TorchProfilerActivity, _is_uri_path
from vllm.config.profiler import (
ProfilerKind,
TorchProfilerActivity,
_is_uri_path,
validate_profile_iteration_bounds,
)
from vllm.config.utils import replace
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -28,14 +35,16 @@ _TRITON_PROTON_3_7_VERSION = Version("3.7.0")
class WorkerProfiler(ABC):
def __init__(self, profiler_config: ProfilerConfig) -> None:
self._delay_iters = profiler_config.delay_iterations
self._default_delay_iters = profiler_config.delay_iterations
self._delay_iters = self._default_delay_iters
if self._delay_iters > 0:
logger.info_once(
"GPU profiling will start "
f"{self._delay_iters} steps after start_profile."
)
self._max_iters = profiler_config.max_iterations
self._default_max_iters = profiler_config.max_iterations
self._max_iters = self._default_max_iters
if self._max_iters > 0:
logger.info_once(
"GPU profiling will stop "
@@ -43,6 +52,9 @@ class WorkerProfiler(ABC):
"or when stop_profile is received."
)
# Serializes start()/stop(), which AsyncLLM may call from different threads.
self._lifecycle_lock = threading.Lock()
# Track when the profiler gets triggered by start_profile
self._active_iteration_count = 0
self._active = False
@@ -88,17 +100,32 @@ class WorkerProfiler(ABC):
logger.warning("Failed to stop profiler: %s", e)
self._running = False # Always mark as not running, assume stop worked
def start(self) -> None:
def start(
self,
*,
delay_iterations: int | None = None,
max_iterations: int | None = None,
) -> None:
"""Attempt to start the profiler, accounting for delayed starts."""
if self._active:
logger.debug(
"start_profile received when profiler is already active. "
"Ignoring request."
with self._lifecycle_lock:
if self._active:
logger.debug(
"start_profile received when profiler is already active. "
"Ignoring request."
)
return
validate_profile_iteration_bounds(delay_iterations, max_iterations)
self._delay_iters = (
self._default_delay_iters
if delay_iterations is None
else delay_iterations
)
return
self._active = True
if self._delay_iters == 0:
self._call_start()
self._max_iters = (
self._default_max_iters if max_iterations is None else max_iterations
)
self._active = True
if self._delay_iters == 0:
self._call_start()
def step(self) -> None:
"""Update the profiler state at each worker step,
@@ -148,17 +175,19 @@ class WorkerProfiler(ABC):
def stop(self) -> None:
"""Attempt to stop the profiler, accounting for overlapped calls."""
if not self._active:
logger.debug(
"stop_profile received when profiler is not active. Ignoring request."
)
return
self._active = False
self._active_iteration_count = 0
self._profiling_for_iters = 0
with self._lifecycle_lock:
if not self._active:
logger.debug(
"stop_profile received when profiler is not active. "
"Ignoring request."
)
return
self._active = False
self._active_iteration_count = 0
self._profiling_for_iters = 0
if self._running:
self._call_stop()
if self._running:
self._call_stop()
def shutdown(self) -> None:
"""Ensure profiler is stopped when shutting down."""
@@ -175,6 +204,10 @@ class WorkerProfiler(ABC):
"""Observe graph creation for backends that attribute replay activity."""
return nullcontext()
def set_output_name(self, worker_name: str) -> None:
"""Set the output name for the next profiling session."""
return None
def annotate_context_manager(self, name: str):
"""Return a context manager to annotate profiler traces."""
return nullcontext()
@@ -202,6 +235,10 @@ class TorchProfilerWrapper(WorkerProfiler):
self.local_rank = local_rank
self.profiler_config = profiler_config
torch_profiler_trace_dir = profiler_config.torch_profiler_dir
self._torch_profiler_trace_dir = torch_profiler_trace_dir
self._torch_profiler_use_gzip = profiler_config.torch_profiler_use_gzip
self._worker_name = worker_name
self._custom_trace_handler = on_trace_ready is not None
if local_rank in (None, 0):
logger.info_once(
"Torch profiling enabled. Traces will be saved to: %s",
@@ -276,6 +313,21 @@ class TorchProfilerWrapper(WorkerProfiler):
self._warmup_steps_remaining = self._initial_warmup_steps_remaining
self._version_metadata_added = False
@override
def set_output_name(self, worker_name: str) -> None:
if self._active:
return
self._worker_name = worker_name
if self._custom_trace_handler:
return
self._profiler_kwargs["on_trace_ready"] = (
torch.profiler.tensorboard_trace_handler(
self._torch_profiler_trace_dir,
worker_name=worker_name,
use_gzip=self._torch_profiler_use_gzip,
)
)
def _build_profiler_table(
self,
sort_key: str,
@@ -293,13 +345,15 @@ class TorchProfilerWrapper(WorkerProfiler):
row_limit=row_limit,
)
def _write_profiler_table(self, rank: int, table: str) -> None:
def _write_profiler_table(self, table: str) -> None:
profiler_dir = self.profiler_config.torch_profiler_dir
# Skip file write for URI paths (gs://, s3://, etc.)
# as standard file I/O doesn't work with URI schemes
if not _is_uri_path(profiler_dir):
profiler_out_file = f"{profiler_dir}/profiler_out_{rank}.txt"
profiler_out_file = os.path.join(
profiler_dir, f"{self._worker_name}.profiler_out.txt"
)
with open(profiler_out_file, "w") as f:
print(table, file=f)
@@ -344,7 +398,7 @@ class TorchProfilerWrapper(WorkerProfiler):
rank = self.local_rank
if self.dump_device_time_total:
table = self._build_profiler_table(sort_key="self_device_time_total")
self._write_profiler_table(rank, table)
self._write_profiler_table(table)
# only print profiler results on rank 0
if rank == 0:
@@ -354,7 +408,7 @@ class TorchProfilerWrapper(WorkerProfiler):
table = self._build_profiler_table(
sort_key="self_cpu_time_total", row_limit=50
)
self._write_profiler_table(rank, table)
self._write_profiler_table(table)
# only print profiler results on rank 0
if rank == 0:
@@ -494,6 +548,7 @@ class ProtonProfilerWrapper(WorkerProfiler):
def has_cuda_graph_session(self) -> bool:
return self._graph_session
@override
def set_output_name(self, worker_name: str) -> None:
"""Set the next run's output name after startup graph capture."""
if self._active:
@@ -701,6 +756,29 @@ def create_worker_profiler(
return ProtonProfilerWrapper(profiler_config, worker_name=worker_name)
def create_frontend_profiler(
profiler_config: ProfilerConfig, *, worker_name: str
) -> TorchProfilerWrapper:
"""Create the CPU-only frontend profiler, which runs for the whole session
rather than following engine iterations."""
logger.info(
"Torch profiler enabled. AsyncLLM CPU traces will be collected under %s",
profiler_config.torch_profiler_dir,
)
return TorchProfilerWrapper(
replace(
profiler_config,
delay_iterations=0,
max_iterations=0,
wait_iterations=0,
warmup_iterations=0,
),
worker_name=worker_name,
local_rank=0,
activities=["CPU"],
)
def create_graph_capture_profiler(
profiler_config: ProfilerConfig, global_rank: int
) -> WorkerProfiler | None: