[Profiler] Unify platform-aware torch profiling (#57460)

This commit is contained in:
czhu-cohere
2026-09-20 19:17:23 -07:00
committed by GitHub
parent 6dfd87f596
commit 3918f3c5a3
9 changed files with 408 additions and 110 deletions
+4
View File
@@ -22,6 +22,10 @@ To use the `torch.profiler` module, set the `profiler` entry to `'torch'` and `t
- `torch_profiler_with_flops` to enable recording FLOPs, off by default
- `torch_profiler_use_gzip` to control gzip-compressing profiling files, on by default
- `torch_profiler_dump_cuda_time_total` to control dumping and printing the aggregated CUDA self time table, on by default
- `torch_profiler_activities` to select worker activities. Defaults are
platform-specific: `["CPU"]` on CPU, `["CPU", "CUDA"]` on NVIDIA GPUs, and
`["CPU", "XPU"]` on Intel GPUs. Selecting `["CUDA"]` omits CPU annotations
and the AsyncLLM CPU trace to reduce profiling overhead and trace size.
When using `vllm bench serve`, you can enable profiling by passing the `--profile` flag.
+30 -7
View File
@@ -60,13 +60,7 @@ VISION_PROMPT = {
}
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
vllm_config = MagicMock()
vllm_config.observability_config.otlp_traces_endpoint = None
vllm_config.scheduler_config.stream_interval = 1
vllm_config.profiler_config.profiler = "cuda"
vllm_config.profiler_config.ignore_frontend = False
def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
renderer = MagicMock()
engine_core = MagicMock()
engine_core.profile_async = AsyncMock()
@@ -90,6 +84,16 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
"make_async_mp_client",
MagicMock(return_value=engine_core),
)
return engine_core
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
vllm_config = MagicMock()
vllm_config.observability_config.otlp_traces_endpoint = None
vllm_config.scheduler_config.stream_interval = 1
vllm_config.profiler_config.profiler = "cuda"
vllm_config.profiler_config.ignore_frontend = False
engine_core = _mock_async_llm_dependencies(monkeypatch)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
@@ -103,6 +107,25 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
monkeypatch: pytest.MonkeyPatch,
):
vllm_config = MagicMock()
vllm_config.observability_config.otlp_traces_endpoint = None
vllm_config.scheduler_config.stream_interval = 1
vllm_config.profiler_config.profiler = "torch"
vllm_config.profiler_config.ignore_frontend = False
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
_mock_async_llm_dependencies(monkeypatch)
profiler = MagicMock()
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
assert engine.profiler is None
profiler.assert_not_called()
async def generate(
engine: AsyncLLM,
request_id: str,
+231 -10
View File
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock, call, patch
from uuid import UUID
import pytest
import torch
from pydantic import ValidationError
from vllm.config import (
@@ -17,10 +18,17 @@ from vllm.config import (
)
from vllm.config.profiler import _is_uri_path
from vllm.platforms import current_platform
from vllm.profiler.wrapper import ProtonProfilerWrapper, WorkerProfiler
from vllm.profiler.wrapper import (
ProtonProfilerWrapper,
TorchProfilerWrapper,
WorkerProfiler,
create_worker_profiler,
validate_worker_profiler_config,
)
from vllm.v1.core.sched.output import CachedRequestData
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.gpu_worker import Worker
from vllm.v1.worker.xpu_worker import XPUWorker
class ConcreteWorkerProfiler(WorkerProfiler):
@@ -51,6 +59,213 @@ def default_profiler_config():
)
def test_torch_profiler_rebuilds_one_shot_profiler_each_round(tmp_path):
config = ProfilerConfig(
profiler="torch",
torch_profiler_dir=str(tmp_path),
torch_profiler_activities=["CUDA"],
torch_profiler_dump_cuda_time_total=False,
warmup_iterations=2,
)
profilers = [MagicMock(), MagicMock()]
for profiler in profilers:
profiler.profiler = MagicMock()
with patch(
"vllm.profiler.wrapper.torch.profiler.profile", side_effect=profilers
) as profile:
wrapper = TorchProfilerWrapper(
config, worker_name="worker", local_rank=0, activities=["CUDA"]
)
profile.assert_not_called()
wrapper.start()
assert not wrapper.should_annotate
assert not wrapper._profiler_step()
wrapper.stop()
wrapper.start()
assert not wrapper._profiler_step()
wrapper.stop()
assert profile.call_count == 2
assert profile.call_args.kwargs["activities"] == [
torch.profiler.ProfilerActivity.CUDA
]
for profiler in profilers:
profiler.start.assert_called_once_with()
profiler.step.assert_called_once_with()
profiler.stop.assert_called_once_with()
def test_torch_profiler_records_each_profile_round(tmp_path):
traces: list[torch.profiler.profile] = []
wrapper = TorchProfilerWrapper(
ProfilerConfig(
profiler="torch",
torch_profiler_dir=str(tmp_path),
torch_profiler_dump_cuda_time_total=False,
),
worker_name="worker",
local_rank=1,
activities=["CPU"],
on_trace_ready=traces.append,
)
for run in range(2):
wrapper.start()
with torch.profiler.record_function(f"run_{run}"):
pass
wrapper.stop()
assert len(traces) == 2
for run, trace in enumerate(traces):
assert {
event.name for event in trace.events() if event.name.startswith("run_")
} == {f"run_{run}"}
@pytest.mark.parametrize(
"activities", [["CPU", "CUDA"], ["CUDA"], ["CPU", "XPU"], ["XPU"]]
)
@pytest.mark.parametrize("dump_device_time", [True, False])
def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device_time):
"""Device summaries honor the dump option on both CUDA and XPU."""
config = ProfilerConfig(
profiler="torch",
torch_profiler_dir=str(tmp_path),
torch_profiler_dump_cuda_time_total=dump_device_time,
)
with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile:
profile.return_value.key_averages.return_value.table.return_value = (
"device times"
)
wrapper = TorchProfilerWrapper(
config, worker_name="worker", local_rank=0, activities=activities
)
wrapper.start()
wrapper.stop()
summary = tmp_path / "profiler_out_0.txt"
assert summary.exists() == dump_device_time
assert ("device times" in capsys.readouterr().out) == dump_device_time
if dump_device_time:
assert summary.read_text() == "device times\n"
@pytest.mark.parametrize(
"activities",
[[], ["CPU", "CPU"], ["INVALID"]],
)
def test_torch_profiler_activities_reject_invalid_values(activities):
with pytest.raises(ValueError, match="torch_profiler_activities"):
ProfilerConfig(
profiler="torch",
torch_profiler_dir="/tmp/mock",
torch_profiler_activities=activities,
)
def test_torch_profiler_activities_require_torch_profiler():
with pytest.raises(ValueError, match="only applicable"):
ProfilerConfig(torch_profiler_activities=["CPU"])
@pytest.mark.parametrize(
("device_type", "activities"),
[
("cuda", ["XPU"]),
("xpu", ["CUDA"]),
("cpu", ["CUDA"]),
("cpu", ["XPU"]),
],
)
def test_worker_rejects_unsupported_activities_at_startup(device_type, activities):
config = ProfilerConfig(
profiler="torch",
torch_profiler_dir="/tmp/mock",
torch_profiler_activities=activities,
)
with (
patch.object(current_platform, "device_type", device_type),
pytest.raises(ValueError, match="Unsupported torch profiler activities"),
):
validate_worker_profiler_config(config)
@pytest.mark.parametrize("device_type", ["cpu", "xpu"])
def test_worker_rejects_cuda_profiler_on_other_devices(device_type):
with (
patch.object(current_platform, "device_type", device_type),
pytest.raises(ValueError, match="Unsupported profiler type"),
):
validate_worker_profiler_config(ProfilerConfig(profiler="cuda"))
@pytest.mark.parametrize(
("device_type", "activities", "expected"),
[
("cuda", None, ("CPU", "CUDA")),
("xpu", None, ("CPU", "XPU")),
("cpu", None, ("CPU",)),
("cuda", ["CUDA"], ("CUDA",)),
("xpu", ["XPU"], ("XPU",)),
("cuda", ["CPU"], ("CPU",)),
("xpu", ["CPU"], ("CPU",)),
("cpu", ["CPU"], ("CPU",)),
],
)
def test_worker_creates_platform_torch_profiler(device_type, activities, expected):
config = ProfilerConfig(
profiler="torch",
torch_profiler_dir="/tmp/mock",
torch_profiler_activities=activities,
)
with (
patch.object(current_platform, "device_type", device_type),
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
):
validate_worker_profiler_config(config)
profiler = create_worker_profiler(
config,
worker_name="rank0",
local_rank=0,
)
assert profiler is wrapper.return_value
wrapper.assert_called_once_with(
config,
worker_name="rank0",
local_rank=0,
activities=expected,
)
@pytest.mark.parametrize("worker_type", [Worker, XPUWorker])
def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type):
worker = object.__new__(worker_type)
worker.rank = 0
worker.local_rank = 0
worker.profiler = None
worker.profiler_config = ProfilerConfig(
profiler="torch", torch_profiler_dir="/tmp/mock"
)
with (
patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"),
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
):
worker.profile()
worker.profile(is_start=False)
worker.profile()
assert worker.profiler is wrapper.return_value
wrapper.assert_called_once()
assert wrapper.return_value.start.call_count == 2
wrapper.return_value.stop.assert_called_once_with()
def test_immediate_start_stop(default_profiler_config):
"""Test standard start without delay."""
profiler = ConcreteWorkerProfiler(default_profiler_config)
@@ -318,13 +533,17 @@ class TestAnnotateProfile:
"execute_5_context_1(sq4sk4sqsq16sqsk16)_generation_1(sq1sk11sqsq1sqsk11)"
)
def test_skips_annotation_work_after_profiler_stops(self):
def test_skips_annotation_work_when_profiler_does_not_annotate(self):
worker = MagicMock()
worker.profiler.is_running = False
worker.profiler.should_annotate = False
context = Worker.annotate_profile(worker, scheduler_output=None)
with patch(
"vllm.v1.worker.gpu_worker.compute_iteration_details"
) as compute_iteration_details:
context = Worker.annotate_profile(worker, scheduler_output=None)
worker.profiler.step.assert_called_once_with()
compute_iteration_details.assert_not_called()
worker.profiler.annotate_context_manager.assert_not_called()
assert isinstance(context, nullcontext)
@@ -837,17 +1056,18 @@ class TestProtonProfilerWrapper:
@_requires_cuda_for_proton
def test_gpu_worker_creates_proton_profiler():
worker = MagicMock()
worker = object.__new__(Worker)
worker.rank = 1
worker.local_rank = 1
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config = MagicMock(profiler="proton")
with (
patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank1",
),
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
):
Worker.profile(worker)
@@ -857,17 +1077,18 @@ def test_gpu_worker_creates_proton_profiler():
@_requires_cuda_for_proton
def test_gpu_worker_recreates_proton_profiler_for_each_run():
worker = MagicMock()
worker = object.__new__(Worker)
worker.rank = 1
worker.local_rank = 1
worker.profiler = None
worker.profiler_config.profiler = "proton"
worker.profiler_config = MagicMock(profiler="proton")
with (
patch(
"vllm.distributed.utils.get_worker_rank_suffix",
return_value="rank1",
),
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
):
wrapper.return_value.has_cuda_graph_session = False
Worker.profile(worker, profile_prefix="first")
+25 -1
View File
@@ -14,6 +14,7 @@ from vllm.utils.hashing import safe_hash
logger = init_logger(__name__)
ProfilerKind = Literal["torch", "cuda", "proton"]
TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"]
ProtonBackend = Literal["cupti"]
ProtonContext = Literal["shadow", "python"]
ProtonData = Literal["tree", "trace"]
@@ -51,6 +52,12 @@ class ProfilerConfig:
worker's traces (CPU & GPU) will be saved under this directory. Note that
it must be an absolute path."""
torch_profiler_activities: list[TorchProfilerActivity] | None = Field(
default=None, min_length=1
)
"""Activities recorded by workers using the torch profiler. When unset,
each worker uses its platform default: CPU; CPU and CUDA; or CPU and XPU."""
proton_profiler_dir: str = ""
"""Directory to save Triton Proton profiles. Each worker writes a
separate rank-qualified file."""
@@ -170,7 +177,24 @@ class ProfilerConfig:
@model_validator(mode="after")
def _validate_profiler_config(self) -> Self:
has_delay_or_limit = self.delay_iterations > 0 or self.max_iterations > 0
if self.profiler == "torch" and has_delay_or_limit and not self.ignore_frontend:
activities = self.torch_profiler_activities
if activities is not None:
if self.profiler != "torch":
raise ValueError(
"torch_profiler_activities is only applicable when profiler "
"is set to 'torch'"
)
if len(activities) != len(set(activities)):
raise ValueError(
"torch_profiler_activities must not contain duplicates"
)
records_cpu_activity = activities is None or "CPU" in activities
if (
self.profiler == "torch"
and has_delay_or_limit
and not self.ignore_frontend
and records_cpu_activity
):
logger.warning_once(
"Using 'torch' profiler with delay_iterations or max_iterations "
"while ignore_frontend is False may result in high overhead."
+106 -11
View File
@@ -6,10 +6,9 @@ import inspect
import json
import os
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from collections.abc import Callable, Iterator, Sequence
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
from glob import glob
from typing import Literal
from uuid import uuid4
import torch
@@ -18,7 +17,7 @@ from typing_extensions import override
import vllm.version
from vllm.config import ProfilerConfig
from vllm.config.profiler import _is_uri_path
from vllm.config.profiler import ProfilerKind, TorchProfilerActivity, _is_uri_path
from vllm.logger import init_logger
from vllm.platforms import current_platform
@@ -57,6 +56,11 @@ class WorkerProfiler(ABC):
"""Whether the underlying profiler is currently collecting data."""
return self._running
@property
def should_annotate(self) -> bool:
"""Whether worker iterations should receive profiler annotations."""
return self._running
@abstractmethod
def _start(self) -> None:
"""Start the profiler."""
@@ -176,7 +180,6 @@ class WorkerProfiler(ABC):
return nullcontext()
TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"]
TorchProfilerActivityMap = {
"CPU": torch.profiler.ProfilerActivity.CPU,
"CUDA": torch.profiler.ProfilerActivity.CUDA,
@@ -191,7 +194,7 @@ class TorchProfilerWrapper(WorkerProfiler):
profiler_config: ProfilerConfig,
worker_name: str,
local_rank: int,
activities: list[TorchProfilerActivity],
activities: Sequence[TorchProfilerActivity],
on_trace_ready: Callable[[torch.profiler.profile], None] | None = None,
) -> None:
super().__init__(profiler_config)
@@ -224,7 +227,12 @@ class TorchProfilerWrapper(WorkerProfiler):
use_gzip=profiler_config.torch_profiler_use_gzip,
)
self.dump_cpu_time_total = "CPU" in activities and len(activities) == 1
self._records_cpu_activity = "CPU" in activities
self.dump_device_time_total = (
any(activity != "CPU" for activity in activities)
and profiler_config.torch_profiler_dump_cuda_time_total
)
self.dump_cpu_time_total = self._records_cpu_activity and len(activities) == 1
# Create profiler schedule if warmup or wait iterations are configured
profiler_schedule = None
@@ -244,7 +252,7 @@ class TorchProfilerWrapper(WorkerProfiler):
profiler_config.active_iterations,
)
self.profiler = torch.profiler.profile(
self._profiler_kwargs = dict(
activities=[TorchProfilerActivityMap[activity] for activity in activities],
schedule=profiler_schedule,
record_shapes=profiler_config.torch_profiler_record_shapes,
@@ -253,6 +261,7 @@ class TorchProfilerWrapper(WorkerProfiler):
with_flops=profiler_config.torch_profiler_with_flops,
on_trace_ready=trace_handler,
)
self.profiler: torch.profiler.profile
# Track if we're using a schedule (need to call step())
self._uses_schedule = profiler_schedule is not None
@@ -260,10 +269,11 @@ class TorchProfilerWrapper(WorkerProfiler):
# Subtract 1 because profiler.start() already consumes step 0
# (WAIT or WARMUP), so only wait + warmup - 1 non-active steps
# remain to be advanced through via profiler.step() calls.
self._warmup_steps_remaining = max(
self._initial_warmup_steps_remaining = max(
profiler_config.wait_iterations + profiler_config.warmup_iterations - 1,
0,
)
self._warmup_steps_remaining = self._initial_warmup_steps_remaining
self._version_metadata_added = False
def _build_profiler_table(
@@ -319,6 +329,9 @@ class TorchProfilerWrapper(WorkerProfiler):
@override
def _start(self) -> None:
self.profiler = torch.profiler.profile(**self._profiler_kwargs)
self._warmup_steps_remaining = self._initial_warmup_steps_remaining
self._version_metadata_added = False
self.profiler.start()
# No-schedule case: Kineto is live immediately. With a schedule this
# no-ops and _profiler_step stamps it once WAIT ends.
@@ -328,10 +341,9 @@ class TorchProfilerWrapper(WorkerProfiler):
def _stop(self) -> None:
self.profiler.stop()
profiler_config = self.profiler_config
rank = self.local_rank
if profiler_config.torch_profiler_dump_cuda_time_total:
table = self._build_profiler_table(sort_key="self_cuda_time_total")
if self.dump_device_time_total:
table = self._build_profiler_table(sort_key="self_device_time_total")
self._write_profiler_table(rank, table)
# only print profiler results on rank 0
@@ -367,8 +379,15 @@ class TorchProfilerWrapper(WorkerProfiler):
return False
return True
@property
@override
def should_annotate(self) -> bool:
return self._running and self._records_cpu_activity
@override
def annotate_context_manager(self, name: str):
if not self.should_annotate:
return nullcontext()
return torch.profiler.record_function(name)
@@ -606,6 +625,82 @@ class CudaProfilerWrapper(WorkerProfiler):
return torch.cuda.nvtx.range(name)
_DEFAULT_TORCH_PROFILER_ACTIVITIES: dict[str, tuple[TorchProfilerActivity, ...]] = {
"cpu": ("CPU",),
"cuda": ("CPU", "CUDA"),
"xpu": ("CPU", "XPU"),
}
_SUPPORTED_TORCH_PROFILER_ACTIVITIES = {
device: frozenset(activities)
for device, activities in _DEFAULT_TORCH_PROFILER_ACTIVITIES.items()
}
_SUPPORTED_PROFILER_KINDS: dict[str, frozenset[ProfilerKind]] = {
"cpu": frozenset(("torch",)),
"cuda": frozenset(("torch", "cuda", "proton")),
"xpu": frozenset(("torch",)),
}
def validate_worker_profiler_config(profiler_config: ProfilerConfig) -> None:
"""Validate profiler selections against the worker's capabilities."""
profiler_type = profiler_config.profiler
if profiler_type is None:
return
device_type = current_platform.device_type
if device_type not in _SUPPORTED_PROFILER_KINDS:
raise ValueError(f"Unsupported profiler device type: {device_type}")
supported_kinds = _SUPPORTED_PROFILER_KINDS[device_type]
if profiler_type not in supported_kinds:
supported_names = ", ".join(sorted(supported_kinds))
raise ValueError(
f"Unsupported profiler type for {device_type}: "
f"{profiler_type}. Supported profiler types: {supported_names}."
)
if profiler_type == "torch":
default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[device_type]
supported_activities = _SUPPORTED_TORCH_PROFILER_ACTIVITIES[device_type]
configured = profiler_config.torch_profiler_activities
activities = default_activities if configured is None else configured
unsupported = set(activities) - supported_activities
if unsupported:
unsupported_names = ", ".join(sorted(unsupported))
supported_names = ", ".join(sorted(supported_activities))
raise ValueError(
f"Unsupported torch profiler activities for "
f"{device_type}: {unsupported_names}. "
f"Supported activities: {supported_names}."
)
def create_worker_profiler(
profiler_config: ProfilerConfig,
*,
worker_name: str,
local_rank: int,
) -> WorkerProfiler:
"""Create a profiler using a validated config and platform defaults."""
profiler_type = profiler_config.profiler
if profiler_type == "torch":
default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[
current_platform.device_type
]
configured = profiler_config.torch_profiler_activities
logger.debug("Starting torch profiler with trace name: %s", worker_name)
return TorchProfilerWrapper(
profiler_config,
worker_name=worker_name,
local_rank=local_rank,
activities=default_activities if configured is None else tuple(configured),
)
if profiler_type == "cuda":
logger.debug("Starting CUDA profiler")
return CudaProfilerWrapper(profiler_config)
assert profiler_type == "proton", f"Unknown profiler type: {profiler_type}"
logger.debug("Starting Proton profiler with trace name: %s", worker_name)
return ProtonProfilerWrapper(profiler_config, worker_name=worker_name)
def create_graph_capture_profiler(
profiler_config: ProfilerConfig, global_rank: int
) -> WorkerProfiler | None:
+2
View File
@@ -208,9 +208,11 @@ class AsyncLLM(EngineClient):
pass
self.profiler = profiler
configured_activities = vllm_config.profiler_config.torch_profiler_activities
if (
vllm_config.profiler_config.profiler == "torch"
and not vllm_config.profiler_config.ignore_frontend
and (configured_activities is None or "CPU" in configured_activities)
):
profiler_dir = vllm_config.profiler_config.torch_profiler_dir
logger.info(
-22
View File
@@ -7,7 +7,6 @@ import vllm.v1.worker.cpu.shm # noqa # isort: skip
import math
import os
import sys
from typing import Any
import psutil
import torch
@@ -16,7 +15,6 @@ from vllm import envs
from vllm.config import CompilationMode, VllmConfig
from vllm.logger import init_logger
from vllm.platforms import CpuArchEnum, current_platform
from vllm.profiler.wrapper import TorchProfilerWrapper
from vllm.utils.cpu_resource_utils import (
get_allowed_cpu_list,
get_memory_node_info,
@@ -106,18 +104,6 @@ class CPUWorker(Worker):
self.parallel_config.disable_custom_all_reduce = True
# Torch profiler. Enabled and configured through profiler_config.
self.profiler: Any | None = None
profiler_config = vllm_config.profiler_config
if profiler_config.profiler == "torch":
worker_name = f"{vllm_config.instance_id}-rank-{self.rank}"
self.profiler = TorchProfilerWrapper(
profiler_config,
worker_name=worker_name,
local_rank=self.local_rank,
activities=["CPU"],
)
def init_device(self):
self.device = torch.device("cpu")
@@ -287,11 +273,3 @@ class CPUWorker(Worker):
language_model=self.compilation_config.compilation_time,
encoder=self.compilation_config.encoder_compilation_time,
)
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
if self.profiler is None:
raise RuntimeError("Profiler is not enabled.")
if is_start:
self.profiler.start()
else:
self.profiler.stop()
+10 -31
View File
@@ -61,10 +61,9 @@ from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
from vllm.platforms import current_platform
from vllm.profiler.wrapper import (
CudaProfilerWrapper,
ProtonProfilerWrapper,
TorchProfilerWrapper,
create_graph_capture_profiler,
create_worker_profiler,
validate_worker_profiler_config,
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
@@ -222,6 +221,7 @@ class Worker(WorkerBase):
# so we have all the information needed for proper trace naming.
self.profiler: Any | None = None
self.profiler_config = vllm_config.profiler_config
validate_worker_profiler_config(self.profiler_config)
self.use_v2_model_runner = vllm_config.use_v2_model_runner
@@ -1038,7 +1038,7 @@ class Worker(WorkerBase):
return nullcontext()
self.profiler.step()
if not self.profiler.is_running:
if not self.profiler.should_annotate:
return nullcontext()
iteration_details = compute_iteration_details(scheduler_output)
@@ -1262,7 +1262,6 @@ class Worker(WorkerBase):
)
if is_start:
profiler_type = self.profiler_config.profiler
# Generate the trace name by combining prefix with comprehensive rank suffix
from vllm.distributed.utils import get_worker_rank_suffix
@@ -1274,36 +1273,16 @@ class Worker(WorkerBase):
else:
trace_name = rank_suffix
if profiler_type == "proton" and self.profiler is not None:
if self.profiler_config.profiler == "proton" and self.profiler is not None:
self.profiler.set_output_name(trace_name)
# Create the profiler wrapper only on the first start call
if self.profiler is None:
if profiler_type == "torch":
self.profiler = TorchProfilerWrapper(
self.profiler_config,
worker_name=trace_name,
local_rank=self.local_rank,
activities=["CPU", "CUDA"],
)
logger.debug(
"Starting torch profiler with trace name: %s", trace_name
)
elif profiler_type == "cuda":
self.profiler = CudaProfilerWrapper(self.profiler_config)
logger.debug("Starting CUDA profiler")
elif profiler_type == "proton":
self.profiler = ProtonProfilerWrapper(
self.profiler_config, worker_name=trace_name
)
logger.debug(
"Starting Proton profiler with trace name: %s", trace_name
)
else:
# Config validation should prevent this code being reached
raise ValueError(
f"Invalid profiler value of {self.profiler_config.profiler}"
)
self.profiler = create_worker_profiler(
self.profiler_config,
worker_name=trace_name,
local_rank=self.local_rank,
)
self.profiler.start()
else:
-28
View File
@@ -8,7 +8,6 @@ import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.profiler.wrapper import TorchProfilerWrapper
from vllm.utils.mem_utils import MemorySnapshot, format_gib
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.utils import report_usage_stats
@@ -225,33 +224,6 @@ class XPUWorker(Worker):
# If usage stat is enabled, collect relevant info.
report_usage_stats(self.vllm_config)
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
if self.profiler_config is None or self.profiler_config.profiler is None:
raise RuntimeError(
"Profiling is not enabled. Please set --profiler-config to enable "
"profiling. Example: "
"'--profiler-config.profiler=torch --profiler-config.torch_profiler_dir"
"=YOUR_DIR_PATH_TO_DUMP_TRACE'"
)
if is_start and self.profiler is None:
from vllm.distributed.utils import get_worker_rank_suffix
rank_suffix = get_worker_rank_suffix(global_rank=self.rank)
trace_name = (
f"{profile_prefix}_{rank_suffix}" if profile_prefix else rank_suffix
)
self.profiler = TorchProfilerWrapper(
self.profiler_config,
worker_name=trace_name,
local_rank=self.local_rank,
activities=["CPU", "XPU"],
)
logger.debug("Starting torch profiler with trace name: %s", trace_name)
super().profile(is_start=is_start, profile_prefix=profile_prefix)
def shutdown(self) -> None:
logger.info(
"XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",