From 3918f3c5a3e389fb17d9c0d629811f3745014db0 Mon Sep 17 00:00:00 2001 From: czhu-cohere Date: Sun, 20 Sep 2026 19:17:23 -0700 Subject: [PATCH] [Profiler] Unify platform-aware torch profiling (#57460) --- docs/contributing/profiling.md | 4 + tests/v1/engine/test_async_llm.py | 37 +++- tests/v1/worker/test_gpu_profiler.py | 241 +++++++++++++++++++++++++-- vllm/config/profiler.py | 26 ++- vllm/profiler/wrapper.py | 117 +++++++++++-- vllm/v1/engine/async_llm.py | 2 + vllm/v1/worker/cpu_worker.py | 22 --- vllm/v1/worker/gpu_worker.py | 41 ++--- vllm/v1/worker/xpu_worker.py | 28 ---- 9 files changed, 408 insertions(+), 110 deletions(-) diff --git a/docs/contributing/profiling.md b/docs/contributing/profiling.md index 880d065a2c..86bd68acef 100644 --- a/docs/contributing/profiling.md +++ b/docs/contributing/profiling.md @@ -22,6 +22,10 @@ To use the `torch.profiler` module, set the `profiler` entry to `'torch'` and `t - `torch_profiler_with_flops` to enable recording FLOPs, off by default - `torch_profiler_use_gzip` to control gzip-compressing profiling files, on by default - `torch_profiler_dump_cuda_time_total` to control dumping and printing the aggregated CUDA self time table, on by default +- `torch_profiler_activities` to select worker activities. Defaults are + platform-specific: `["CPU"]` on CPU, `["CPU", "CUDA"]` on NVIDIA GPUs, and + `["CPU", "XPU"]` on Intel GPUs. Selecting `["CUDA"]` omits CPU annotations + and the AsyncLLM CPU trace to reduce profiling overhead and trace size. When using `vllm bench serve`, you can enable profiling by passing the `--profile` flag. diff --git a/tests/v1/engine/test_async_llm.py b/tests/v1/engine/test_async_llm.py index 53a68aa135..cac43d8204 100644 --- a/tests/v1/engine/test_async_llm.py +++ b/tests/v1/engine/test_async_llm.py @@ -60,13 +60,7 @@ VISION_PROMPT = { } -def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch): - vllm_config = MagicMock() - vllm_config.observability_config.otlp_traces_endpoint = None - vllm_config.scheduler_config.stream_interval = 1 - vllm_config.profiler_config.profiler = "cuda" - vllm_config.profiler_config.ignore_frontend = False - +def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch): renderer = MagicMock() engine_core = MagicMock() engine_core.profile_async = AsyncMock() @@ -90,6 +84,16 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc "make_async_mp_client", MagicMock(return_value=engine_core), ) + return engine_core + + +def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch): + vllm_config = MagicMock() + vllm_config.observability_config.otlp_traces_endpoint = None + vllm_config.scheduler_config.stream_interval = 1 + vllm_config.profiler_config.profiler = "cuda" + vllm_config.profiler_config.ignore_frontend = False + engine_core = _mock_async_llm_dependencies(monkeypatch) engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False) @@ -103,6 +107,25 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc engine_core.profile_async.assert_has_awaits([call(True, None), call(False)]) +def test_cuda_only_torch_profiler_skips_frontend_cpu_trace( + monkeypatch: pytest.MonkeyPatch, +): + vllm_config = MagicMock() + vllm_config.observability_config.otlp_traces_endpoint = None + vllm_config.scheduler_config.stream_interval = 1 + vllm_config.profiler_config.profiler = "torch" + vllm_config.profiler_config.ignore_frontend = False + vllm_config.profiler_config.torch_profiler_activities = ["CUDA"] + _mock_async_llm_dependencies(monkeypatch) + profiler = MagicMock() + monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler) + + engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False) + + assert engine.profiler is None + profiler.assert_not_called() + + async def generate( engine: AsyncLLM, request_id: str, diff --git a/tests/v1/worker/test_gpu_profiler.py b/tests/v1/worker/test_gpu_profiler.py index a283b6d932..cb3b177f41 100644 --- a/tests/v1/worker/test_gpu_profiler.py +++ b/tests/v1/worker/test_gpu_profiler.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock, call, patch from uuid import UUID import pytest +import torch from pydantic import ValidationError from vllm.config import ( @@ -17,10 +18,17 @@ from vllm.config import ( ) from vllm.config.profiler import _is_uri_path from vllm.platforms import current_platform -from vllm.profiler.wrapper import ProtonProfilerWrapper, WorkerProfiler +from vllm.profiler.wrapper import ( + ProtonProfilerWrapper, + TorchProfilerWrapper, + WorkerProfiler, + create_worker_profiler, + validate_worker_profiler_config, +) from vllm.v1.core.sched.output import CachedRequestData from vllm.v1.worker.gpu_model_runner import GPUModelRunner from vllm.v1.worker.gpu_worker import Worker +from vllm.v1.worker.xpu_worker import XPUWorker class ConcreteWorkerProfiler(WorkerProfiler): @@ -51,6 +59,213 @@ def default_profiler_config(): ) +def test_torch_profiler_rebuilds_one_shot_profiler_each_round(tmp_path): + config = ProfilerConfig( + profiler="torch", + torch_profiler_dir=str(tmp_path), + torch_profiler_activities=["CUDA"], + torch_profiler_dump_cuda_time_total=False, + warmup_iterations=2, + ) + profilers = [MagicMock(), MagicMock()] + for profiler in profilers: + profiler.profiler = MagicMock() + + with patch( + "vllm.profiler.wrapper.torch.profiler.profile", side_effect=profilers + ) as profile: + wrapper = TorchProfilerWrapper( + config, worker_name="worker", local_rank=0, activities=["CUDA"] + ) + profile.assert_not_called() + + wrapper.start() + assert not wrapper.should_annotate + assert not wrapper._profiler_step() + wrapper.stop() + wrapper.start() + assert not wrapper._profiler_step() + wrapper.stop() + + assert profile.call_count == 2 + assert profile.call_args.kwargs["activities"] == [ + torch.profiler.ProfilerActivity.CUDA + ] + for profiler in profilers: + profiler.start.assert_called_once_with() + profiler.step.assert_called_once_with() + profiler.stop.assert_called_once_with() + + +def test_torch_profiler_records_each_profile_round(tmp_path): + traces: list[torch.profiler.profile] = [] + wrapper = TorchProfilerWrapper( + ProfilerConfig( + profiler="torch", + torch_profiler_dir=str(tmp_path), + torch_profiler_dump_cuda_time_total=False, + ), + worker_name="worker", + local_rank=1, + activities=["CPU"], + on_trace_ready=traces.append, + ) + + for run in range(2): + wrapper.start() + with torch.profiler.record_function(f"run_{run}"): + pass + wrapper.stop() + + assert len(traces) == 2 + for run, trace in enumerate(traces): + assert { + event.name for event in trace.events() if event.name.startswith("run_") + } == {f"run_{run}"} + + +@pytest.mark.parametrize( + "activities", [["CPU", "CUDA"], ["CUDA"], ["CPU", "XPU"], ["XPU"]] +) +@pytest.mark.parametrize("dump_device_time", [True, False]) +def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device_time): + """Device summaries honor the dump option on both CUDA and XPU.""" + config = ProfilerConfig( + profiler="torch", + torch_profiler_dir=str(tmp_path), + torch_profiler_dump_cuda_time_total=dump_device_time, + ) + with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile: + profile.return_value.key_averages.return_value.table.return_value = ( + "device times" + ) + wrapper = TorchProfilerWrapper( + config, worker_name="worker", local_rank=0, activities=activities + ) + wrapper.start() + wrapper.stop() + + summary = tmp_path / "profiler_out_0.txt" + assert summary.exists() == dump_device_time + assert ("device times" in capsys.readouterr().out) == dump_device_time + if dump_device_time: + assert summary.read_text() == "device times\n" + + +@pytest.mark.parametrize( + "activities", + [[], ["CPU", "CPU"], ["INVALID"]], +) +def test_torch_profiler_activities_reject_invalid_values(activities): + with pytest.raises(ValueError, match="torch_profiler_activities"): + ProfilerConfig( + profiler="torch", + torch_profiler_dir="/tmp/mock", + torch_profiler_activities=activities, + ) + + +def test_torch_profiler_activities_require_torch_profiler(): + with pytest.raises(ValueError, match="only applicable"): + ProfilerConfig(torch_profiler_activities=["CPU"]) + + +@pytest.mark.parametrize( + ("device_type", "activities"), + [ + ("cuda", ["XPU"]), + ("xpu", ["CUDA"]), + ("cpu", ["CUDA"]), + ("cpu", ["XPU"]), + ], +) +def test_worker_rejects_unsupported_activities_at_startup(device_type, activities): + config = ProfilerConfig( + profiler="torch", + torch_profiler_dir="/tmp/mock", + torch_profiler_activities=activities, + ) + + with ( + patch.object(current_platform, "device_type", device_type), + pytest.raises(ValueError, match="Unsupported torch profiler activities"), + ): + validate_worker_profiler_config(config) + + +@pytest.mark.parametrize("device_type", ["cpu", "xpu"]) +def test_worker_rejects_cuda_profiler_on_other_devices(device_type): + with ( + patch.object(current_platform, "device_type", device_type), + pytest.raises(ValueError, match="Unsupported profiler type"), + ): + validate_worker_profiler_config(ProfilerConfig(profiler="cuda")) + + +@pytest.mark.parametrize( + ("device_type", "activities", "expected"), + [ + ("cuda", None, ("CPU", "CUDA")), + ("xpu", None, ("CPU", "XPU")), + ("cpu", None, ("CPU",)), + ("cuda", ["CUDA"], ("CUDA",)), + ("xpu", ["XPU"], ("XPU",)), + ("cuda", ["CPU"], ("CPU",)), + ("xpu", ["CPU"], ("CPU",)), + ("cpu", ["CPU"], ("CPU",)), + ], +) +def test_worker_creates_platform_torch_profiler(device_type, activities, expected): + config = ProfilerConfig( + profiler="torch", + torch_profiler_dir="/tmp/mock", + torch_profiler_activities=activities, + ) + + with ( + patch.object(current_platform, "device_type", device_type), + patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper, + ): + validate_worker_profiler_config(config) + profiler = create_worker_profiler( + config, + worker_name="rank0", + local_rank=0, + ) + + assert profiler is wrapper.return_value + wrapper.assert_called_once_with( + config, + worker_name="rank0", + local_rank=0, + activities=expected, + ) + + +@pytest.mark.parametrize("worker_type", [Worker, XPUWorker]) +def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type): + worker = object.__new__(worker_type) + worker.rank = 0 + worker.local_rank = 0 + worker.profiler = None + worker.profiler_config = ProfilerConfig( + profiler="torch", torch_profiler_dir="/tmp/mock" + ) + + with ( + patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"), + patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper, + ): + worker.profile() + worker.profile(is_start=False) + worker.profile() + + assert worker.profiler is wrapper.return_value + wrapper.assert_called_once() + assert wrapper.return_value.start.call_count == 2 + wrapper.return_value.stop.assert_called_once_with() + + def test_immediate_start_stop(default_profiler_config): """Test standard start without delay.""" profiler = ConcreteWorkerProfiler(default_profiler_config) @@ -318,13 +533,17 @@ class TestAnnotateProfile: "execute_5_context_1(sq4sk4sqsq16sqsk16)_generation_1(sq1sk11sqsq1sqsk11)" ) - def test_skips_annotation_work_after_profiler_stops(self): + def test_skips_annotation_work_when_profiler_does_not_annotate(self): worker = MagicMock() - worker.profiler.is_running = False + worker.profiler.should_annotate = False - context = Worker.annotate_profile(worker, scheduler_output=None) + with patch( + "vllm.v1.worker.gpu_worker.compute_iteration_details" + ) as compute_iteration_details: + context = Worker.annotate_profile(worker, scheduler_output=None) worker.profiler.step.assert_called_once_with() + compute_iteration_details.assert_not_called() worker.profiler.annotate_context_manager.assert_not_called() assert isinstance(context, nullcontext) @@ -837,17 +1056,18 @@ class TestProtonProfilerWrapper: @_requires_cuda_for_proton def test_gpu_worker_creates_proton_profiler(): - worker = MagicMock() + worker = object.__new__(Worker) worker.rank = 1 + worker.local_rank = 1 worker.profiler = None - worker.profiler_config.profiler = "proton" + worker.profiler_config = MagicMock(profiler="proton") with ( patch( "vllm.distributed.utils.get_worker_rank_suffix", return_value="rank1", ), - patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper, + patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper, ): Worker.profile(worker) @@ -857,17 +1077,18 @@ def test_gpu_worker_creates_proton_profiler(): @_requires_cuda_for_proton def test_gpu_worker_recreates_proton_profiler_for_each_run(): - worker = MagicMock() + worker = object.__new__(Worker) worker.rank = 1 + worker.local_rank = 1 worker.profiler = None - worker.profiler_config.profiler = "proton" + worker.profiler_config = MagicMock(profiler="proton") with ( patch( "vllm.distributed.utils.get_worker_rank_suffix", return_value="rank1", ), - patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper, + patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper, ): wrapper.return_value.has_cuda_graph_session = False Worker.profile(worker, profile_prefix="first") diff --git a/vllm/config/profiler.py b/vllm/config/profiler.py index f3a6985324..3ade4bceed 100644 --- a/vllm/config/profiler.py +++ b/vllm/config/profiler.py @@ -14,6 +14,7 @@ from vllm.utils.hashing import safe_hash logger = init_logger(__name__) ProfilerKind = Literal["torch", "cuda", "proton"] +TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"] ProtonBackend = Literal["cupti"] ProtonContext = Literal["shadow", "python"] ProtonData = Literal["tree", "trace"] @@ -51,6 +52,12 @@ class ProfilerConfig: worker's traces (CPU & GPU) will be saved under this directory. Note that it must be an absolute path.""" + torch_profiler_activities: list[TorchProfilerActivity] | None = Field( + default=None, min_length=1 + ) + """Activities recorded by workers using the torch profiler. When unset, + each worker uses its platform default: CPU; CPU and CUDA; or CPU and XPU.""" + proton_profiler_dir: str = "" """Directory to save Triton Proton profiles. Each worker writes a separate rank-qualified file.""" @@ -170,7 +177,24 @@ class ProfilerConfig: @model_validator(mode="after") def _validate_profiler_config(self) -> Self: has_delay_or_limit = self.delay_iterations > 0 or self.max_iterations > 0 - if self.profiler == "torch" and has_delay_or_limit and not self.ignore_frontend: + activities = self.torch_profiler_activities + if activities is not None: + if self.profiler != "torch": + raise ValueError( + "torch_profiler_activities is only applicable when profiler " + "is set to 'torch'" + ) + if len(activities) != len(set(activities)): + raise ValueError( + "torch_profiler_activities must not contain duplicates" + ) + records_cpu_activity = activities is None or "CPU" in activities + if ( + self.profiler == "torch" + and has_delay_or_limit + and not self.ignore_frontend + and records_cpu_activity + ): logger.warning_once( "Using 'torch' profiler with delay_iterations or max_iterations " "while ignore_frontend is False may result in high overhead." diff --git a/vllm/profiler/wrapper.py b/vllm/profiler/wrapper.py index bc679407c3..bd9b6c231d 100644 --- a/vllm/profiler/wrapper.py +++ b/vllm/profiler/wrapper.py @@ -6,10 +6,9 @@ import inspect import json import os from abc import ABC, abstractmethod -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress from glob import glob -from typing import Literal from uuid import uuid4 import torch @@ -18,7 +17,7 @@ from typing_extensions import override import vllm.version from vllm.config import ProfilerConfig -from vllm.config.profiler import _is_uri_path +from vllm.config.profiler import ProfilerKind, TorchProfilerActivity, _is_uri_path from vllm.logger import init_logger from vllm.platforms import current_platform @@ -57,6 +56,11 @@ class WorkerProfiler(ABC): """Whether the underlying profiler is currently collecting data.""" return self._running + @property + def should_annotate(self) -> bool: + """Whether worker iterations should receive profiler annotations.""" + return self._running + @abstractmethod def _start(self) -> None: """Start the profiler.""" @@ -176,7 +180,6 @@ class WorkerProfiler(ABC): return nullcontext() -TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"] TorchProfilerActivityMap = { "CPU": torch.profiler.ProfilerActivity.CPU, "CUDA": torch.profiler.ProfilerActivity.CUDA, @@ -191,7 +194,7 @@ class TorchProfilerWrapper(WorkerProfiler): profiler_config: ProfilerConfig, worker_name: str, local_rank: int, - activities: list[TorchProfilerActivity], + activities: Sequence[TorchProfilerActivity], on_trace_ready: Callable[[torch.profiler.profile], None] | None = None, ) -> None: super().__init__(profiler_config) @@ -224,7 +227,12 @@ class TorchProfilerWrapper(WorkerProfiler): use_gzip=profiler_config.torch_profiler_use_gzip, ) - self.dump_cpu_time_total = "CPU" in activities and len(activities) == 1 + self._records_cpu_activity = "CPU" in activities + self.dump_device_time_total = ( + any(activity != "CPU" for activity in activities) + and profiler_config.torch_profiler_dump_cuda_time_total + ) + self.dump_cpu_time_total = self._records_cpu_activity and len(activities) == 1 # Create profiler schedule if warmup or wait iterations are configured profiler_schedule = None @@ -244,7 +252,7 @@ class TorchProfilerWrapper(WorkerProfiler): profiler_config.active_iterations, ) - self.profiler = torch.profiler.profile( + self._profiler_kwargs = dict( activities=[TorchProfilerActivityMap[activity] for activity in activities], schedule=profiler_schedule, record_shapes=profiler_config.torch_profiler_record_shapes, @@ -253,6 +261,7 @@ class TorchProfilerWrapper(WorkerProfiler): with_flops=profiler_config.torch_profiler_with_flops, on_trace_ready=trace_handler, ) + self.profiler: torch.profiler.profile # Track if we're using a schedule (need to call step()) self._uses_schedule = profiler_schedule is not None @@ -260,10 +269,11 @@ class TorchProfilerWrapper(WorkerProfiler): # Subtract 1 because profiler.start() already consumes step 0 # (WAIT or WARMUP), so only wait + warmup - 1 non-active steps # remain to be advanced through via profiler.step() calls. - self._warmup_steps_remaining = max( + self._initial_warmup_steps_remaining = max( profiler_config.wait_iterations + profiler_config.warmup_iterations - 1, 0, ) + self._warmup_steps_remaining = self._initial_warmup_steps_remaining self._version_metadata_added = False def _build_profiler_table( @@ -319,6 +329,9 @@ class TorchProfilerWrapper(WorkerProfiler): @override def _start(self) -> None: + self.profiler = torch.profiler.profile(**self._profiler_kwargs) + self._warmup_steps_remaining = self._initial_warmup_steps_remaining + self._version_metadata_added = False self.profiler.start() # No-schedule case: Kineto is live immediately. With a schedule this # no-ops and _profiler_step stamps it once WAIT ends. @@ -328,10 +341,9 @@ class TorchProfilerWrapper(WorkerProfiler): def _stop(self) -> None: self.profiler.stop() - profiler_config = self.profiler_config rank = self.local_rank - if profiler_config.torch_profiler_dump_cuda_time_total: - table = self._build_profiler_table(sort_key="self_cuda_time_total") + if self.dump_device_time_total: + table = self._build_profiler_table(sort_key="self_device_time_total") self._write_profiler_table(rank, table) # only print profiler results on rank 0 @@ -367,8 +379,15 @@ class TorchProfilerWrapper(WorkerProfiler): return False return True + @property + @override + def should_annotate(self) -> bool: + return self._running and self._records_cpu_activity + @override def annotate_context_manager(self, name: str): + if not self.should_annotate: + return nullcontext() return torch.profiler.record_function(name) @@ -606,6 +625,82 @@ class CudaProfilerWrapper(WorkerProfiler): return torch.cuda.nvtx.range(name) +_DEFAULT_TORCH_PROFILER_ACTIVITIES: dict[str, tuple[TorchProfilerActivity, ...]] = { + "cpu": ("CPU",), + "cuda": ("CPU", "CUDA"), + "xpu": ("CPU", "XPU"), +} +_SUPPORTED_TORCH_PROFILER_ACTIVITIES = { + device: frozenset(activities) + for device, activities in _DEFAULT_TORCH_PROFILER_ACTIVITIES.items() +} +_SUPPORTED_PROFILER_KINDS: dict[str, frozenset[ProfilerKind]] = { + "cpu": frozenset(("torch",)), + "cuda": frozenset(("torch", "cuda", "proton")), + "xpu": frozenset(("torch",)), +} + + +def validate_worker_profiler_config(profiler_config: ProfilerConfig) -> None: + """Validate profiler selections against the worker's capabilities.""" + profiler_type = profiler_config.profiler + if profiler_type is None: + return + device_type = current_platform.device_type + if device_type not in _SUPPORTED_PROFILER_KINDS: + raise ValueError(f"Unsupported profiler device type: {device_type}") + supported_kinds = _SUPPORTED_PROFILER_KINDS[device_type] + if profiler_type not in supported_kinds: + supported_names = ", ".join(sorted(supported_kinds)) + raise ValueError( + f"Unsupported profiler type for {device_type}: " + f"{profiler_type}. Supported profiler types: {supported_names}." + ) + if profiler_type == "torch": + default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[device_type] + supported_activities = _SUPPORTED_TORCH_PROFILER_ACTIVITIES[device_type] + configured = profiler_config.torch_profiler_activities + activities = default_activities if configured is None else configured + unsupported = set(activities) - supported_activities + if unsupported: + unsupported_names = ", ".join(sorted(unsupported)) + supported_names = ", ".join(sorted(supported_activities)) + raise ValueError( + f"Unsupported torch profiler activities for " + f"{device_type}: {unsupported_names}. " + f"Supported activities: {supported_names}." + ) + + +def create_worker_profiler( + profiler_config: ProfilerConfig, + *, + worker_name: str, + local_rank: int, +) -> WorkerProfiler: + """Create a profiler using a validated config and platform defaults.""" + profiler_type = profiler_config.profiler + if profiler_type == "torch": + default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[ + current_platform.device_type + ] + configured = profiler_config.torch_profiler_activities + logger.debug("Starting torch profiler with trace name: %s", worker_name) + return TorchProfilerWrapper( + profiler_config, + worker_name=worker_name, + local_rank=local_rank, + activities=default_activities if configured is None else tuple(configured), + ) + if profiler_type == "cuda": + logger.debug("Starting CUDA profiler") + return CudaProfilerWrapper(profiler_config) + + assert profiler_type == "proton", f"Unknown profiler type: {profiler_type}" + logger.debug("Starting Proton profiler with trace name: %s", worker_name) + return ProtonProfilerWrapper(profiler_config, worker_name=worker_name) + + def create_graph_capture_profiler( profiler_config: ProfilerConfig, global_rank: int ) -> WorkerProfiler | None: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 67b8ac560d..ee913b052f 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -208,9 +208,11 @@ class AsyncLLM(EngineClient): pass self.profiler = profiler + configured_activities = vllm_config.profiler_config.torch_profiler_activities if ( vllm_config.profiler_config.profiler == "torch" and not vllm_config.profiler_config.ignore_frontend + and (configured_activities is None or "CPU" in configured_activities) ): profiler_dir = vllm_config.profiler_config.torch_profiler_dir logger.info( diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index 1badcfe6e8..e36b15300b 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -7,7 +7,6 @@ import vllm.v1.worker.cpu.shm # noqa # isort: skip import math import os import sys -from typing import Any import psutil import torch @@ -16,7 +15,6 @@ from vllm import envs from vllm.config import CompilationMode, VllmConfig from vllm.logger import init_logger from vllm.platforms import CpuArchEnum, current_platform -from vllm.profiler.wrapper import TorchProfilerWrapper from vllm.utils.cpu_resource_utils import ( get_allowed_cpu_list, get_memory_node_info, @@ -106,18 +104,6 @@ class CPUWorker(Worker): self.parallel_config.disable_custom_all_reduce = True - # Torch profiler. Enabled and configured through profiler_config. - self.profiler: Any | None = None - profiler_config = vllm_config.profiler_config - if profiler_config.profiler == "torch": - worker_name = f"{vllm_config.instance_id}-rank-{self.rank}" - self.profiler = TorchProfilerWrapper( - profiler_config, - worker_name=worker_name, - local_rank=self.local_rank, - activities=["CPU"], - ) - def init_device(self): self.device = torch.device("cpu") @@ -287,11 +273,3 @@ class CPUWorker(Worker): language_model=self.compilation_config.compilation_time, encoder=self.compilation_config.encoder_compilation_time, ) - - def profile(self, is_start: bool = True, profile_prefix: str | None = None): - if self.profiler is None: - raise RuntimeError("Profiler is not enabled.") - if is_start: - self.profiler.start() - else: - self.profiler.stop() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 8ced3bdd43..d914bd2dea 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -61,10 +61,9 @@ from vllm.model_executor.warmup.kernel_warmup import kernel_warmup from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory from vllm.platforms import current_platform from vllm.profiler.wrapper import ( - CudaProfilerWrapper, - ProtonProfilerWrapper, - TorchProfilerWrapper, create_graph_capture_profiler, + create_worker_profiler, + validate_worker_profiler_config, ) from vllm.sequence import IntermediateTensors from vllm.tasks import SupportedTask @@ -222,6 +221,7 @@ class Worker(WorkerBase): # so we have all the information needed for proper trace naming. self.profiler: Any | None = None self.profiler_config = vllm_config.profiler_config + validate_worker_profiler_config(self.profiler_config) self.use_v2_model_runner = vllm_config.use_v2_model_runner @@ -1038,7 +1038,7 @@ class Worker(WorkerBase): return nullcontext() self.profiler.step() - if not self.profiler.is_running: + if not self.profiler.should_annotate: return nullcontext() iteration_details = compute_iteration_details(scheduler_output) @@ -1262,7 +1262,6 @@ class Worker(WorkerBase): ) if is_start: - profiler_type = self.profiler_config.profiler # Generate the trace name by combining prefix with comprehensive rank suffix from vllm.distributed.utils import get_worker_rank_suffix @@ -1274,36 +1273,16 @@ class Worker(WorkerBase): else: trace_name = rank_suffix - if profiler_type == "proton" and self.profiler is not None: + if self.profiler_config.profiler == "proton" and self.profiler is not None: self.profiler.set_output_name(trace_name) # Create the profiler wrapper only on the first start call if self.profiler is None: - if profiler_type == "torch": - self.profiler = TorchProfilerWrapper( - self.profiler_config, - worker_name=trace_name, - local_rank=self.local_rank, - activities=["CPU", "CUDA"], - ) - logger.debug( - "Starting torch profiler with trace name: %s", trace_name - ) - elif profiler_type == "cuda": - self.profiler = CudaProfilerWrapper(self.profiler_config) - logger.debug("Starting CUDA profiler") - elif profiler_type == "proton": - self.profiler = ProtonProfilerWrapper( - self.profiler_config, worker_name=trace_name - ) - logger.debug( - "Starting Proton profiler with trace name: %s", trace_name - ) - else: - # Config validation should prevent this code being reached - raise ValueError( - f"Invalid profiler value of {self.profiler_config.profiler}" - ) + self.profiler = create_worker_profiler( + self.profiler_config, + worker_name=trace_name, + local_rank=self.local_rank, + ) self.profiler.start() else: diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index 5c5d125f4b..02cc2c9e87 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -8,7 +8,6 @@ import torch from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.profiler.wrapper import TorchProfilerWrapper from vllm.utils.mem_utils import MemorySnapshot, format_gib from vllm.utils.torch_utils import set_random_seed from vllm.v1.utils import report_usage_stats @@ -225,33 +224,6 @@ class XPUWorker(Worker): # If usage stat is enabled, collect relevant info. report_usage_stats(self.vllm_config) - def profile(self, is_start: bool = True, profile_prefix: str | None = None): - if self.profiler_config is None or self.profiler_config.profiler is None: - raise RuntimeError( - "Profiling is not enabled. Please set --profiler-config to enable " - "profiling. Example: " - "'--profiler-config.profiler=torch --profiler-config.torch_profiler_dir" - "=YOUR_DIR_PATH_TO_DUMP_TRACE'" - ) - - if is_start and self.profiler is None: - from vllm.distributed.utils import get_worker_rank_suffix - - rank_suffix = get_worker_rank_suffix(global_rank=self.rank) - trace_name = ( - f"{profile_prefix}_{rank_suffix}" if profile_prefix else rank_suffix - ) - - self.profiler = TorchProfilerWrapper( - self.profiler_config, - worker_name=trace_name, - local_rank=self.local_rank, - activities=["CPU", "XPU"], - ) - logger.debug("Starting torch profiler with trace name: %s", trace_name) - - super().profile(is_start=is_start, profile_prefix=profile_prefix) - def shutdown(self) -> None: logger.info( "XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",