[Profiler] Unify platform-aware torch profiling (#57460)
This commit is contained in:
@@ -22,6 +22,10 @@ To use the `torch.profiler` module, set the `profiler` entry to `'torch'` and `t
|
||||
- `torch_profiler_with_flops` to enable recording FLOPs, off by default
|
||||
- `torch_profiler_use_gzip` to control gzip-compressing profiling files, on by default
|
||||
- `torch_profiler_dump_cuda_time_total` to control dumping and printing the aggregated CUDA self time table, on by default
|
||||
- `torch_profiler_activities` to select worker activities. Defaults are
|
||||
platform-specific: `["CPU"]` on CPU, `["CPU", "CUDA"]` on NVIDIA GPUs, and
|
||||
`["CPU", "XPU"]` on Intel GPUs. Selecting `["CUDA"]` omits CPU annotations
|
||||
and the AsyncLLM CPU trace to reduce profiling overhead and trace size.
|
||||
|
||||
When using `vllm bench serve`, you can enable profiling by passing the `--profile` flag.
|
||||
|
||||
|
||||
@@ -60,13 +60,7 @@ VISION_PROMPT = {
|
||||
}
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
|
||||
def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
|
||||
renderer = MagicMock()
|
||||
engine_core = MagicMock()
|
||||
engine_core.profile_async = AsyncMock()
|
||||
@@ -90,6 +84,16 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
"make_async_mp_client",
|
||||
MagicMock(return_value=engine_core),
|
||||
)
|
||||
return engine_core
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
@@ -103,6 +107,25 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
|
||||
|
||||
|
||||
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "torch"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
assert engine.profiler is None
|
||||
profiler.assert_not_called()
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncLLM,
|
||||
request_id: str,
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock, call, patch
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config import (
|
||||
@@ -17,10 +18,17 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.config.profiler import _is_uri_path
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import ProtonProfilerWrapper, WorkerProfiler
|
||||
from vllm.profiler.wrapper import (
|
||||
ProtonProfilerWrapper,
|
||||
TorchProfilerWrapper,
|
||||
WorkerProfiler,
|
||||
create_worker_profiler,
|
||||
validate_worker_profiler_config,
|
||||
)
|
||||
from vllm.v1.core.sched.output import CachedRequestData
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
from vllm.v1.worker.xpu_worker import XPUWorker
|
||||
|
||||
|
||||
class ConcreteWorkerProfiler(WorkerProfiler):
|
||||
@@ -51,6 +59,213 @@ def default_profiler_config():
|
||||
)
|
||||
|
||||
|
||||
def test_torch_profiler_rebuilds_one_shot_profiler_each_round(tmp_path):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CUDA"],
|
||||
torch_profiler_dump_cuda_time_total=False,
|
||||
warmup_iterations=2,
|
||||
)
|
||||
profilers = [MagicMock(), MagicMock()]
|
||||
for profiler in profilers:
|
||||
profiler.profiler = MagicMock()
|
||||
|
||||
with patch(
|
||||
"vllm.profiler.wrapper.torch.profiler.profile", side_effect=profilers
|
||||
) as profile:
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="worker", local_rank=0, activities=["CUDA"]
|
||||
)
|
||||
profile.assert_not_called()
|
||||
|
||||
wrapper.start()
|
||||
assert not wrapper.should_annotate
|
||||
assert not wrapper._profiler_step()
|
||||
wrapper.stop()
|
||||
wrapper.start()
|
||||
assert not wrapper._profiler_step()
|
||||
wrapper.stop()
|
||||
|
||||
assert profile.call_count == 2
|
||||
assert profile.call_args.kwargs["activities"] == [
|
||||
torch.profiler.ProfilerActivity.CUDA
|
||||
]
|
||||
for profiler in profilers:
|
||||
profiler.start.assert_called_once_with()
|
||||
profiler.step.assert_called_once_with()
|
||||
profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_torch_profiler_records_each_profile_round(tmp_path):
|
||||
traces: list[torch.profiler.profile] = []
|
||||
wrapper = TorchProfilerWrapper(
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_dump_cuda_time_total=False,
|
||||
),
|
||||
worker_name="worker",
|
||||
local_rank=1,
|
||||
activities=["CPU"],
|
||||
on_trace_ready=traces.append,
|
||||
)
|
||||
|
||||
for run in range(2):
|
||||
wrapper.start()
|
||||
with torch.profiler.record_function(f"run_{run}"):
|
||||
pass
|
||||
wrapper.stop()
|
||||
|
||||
assert len(traces) == 2
|
||||
for run, trace in enumerate(traces):
|
||||
assert {
|
||||
event.name for event in trace.events() if event.name.startswith("run_")
|
||||
} == {f"run_{run}"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities", [["CPU", "CUDA"], ["CUDA"], ["CPU", "XPU"], ["XPU"]]
|
||||
)
|
||||
@pytest.mark.parametrize("dump_device_time", [True, False])
|
||||
def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device_time):
|
||||
"""Device summaries honor the dump option on both CUDA and XPU."""
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_dump_cuda_time_total=dump_device_time,
|
||||
)
|
||||
with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile:
|
||||
profile.return_value.key_averages.return_value.table.return_value = (
|
||||
"device times"
|
||||
)
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="worker", local_rank=0, activities=activities
|
||||
)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
summary = tmp_path / "profiler_out_0.txt"
|
||||
assert summary.exists() == dump_device_time
|
||||
assert ("device times" in capsys.readouterr().out) == dump_device_time
|
||||
if dump_device_time:
|
||||
assert summary.read_text() == "device times\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities",
|
||||
[[], ["CPU", "CPU"], ["INVALID"]],
|
||||
)
|
||||
def test_torch_profiler_activities_reject_invalid_values(activities):
|
||||
with pytest.raises(ValueError, match="torch_profiler_activities"):
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
|
||||
def test_torch_profiler_activities_require_torch_profiler():
|
||||
with pytest.raises(ValueError, match="only applicable"):
|
||||
ProfilerConfig(torch_profiler_activities=["CPU"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities"),
|
||||
[
|
||||
("cuda", ["XPU"]),
|
||||
("xpu", ["CUDA"]),
|
||||
("cpu", ["CUDA"]),
|
||||
("cpu", ["XPU"]),
|
||||
],
|
||||
)
|
||||
def test_worker_rejects_unsupported_activities_at_startup(device_type, activities):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
pytest.raises(ValueError, match="Unsupported torch profiler activities"),
|
||||
):
|
||||
validate_worker_profiler_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_type", ["cpu", "xpu"])
|
||||
def test_worker_rejects_cuda_profiler_on_other_devices(device_type):
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
pytest.raises(ValueError, match="Unsupported profiler type"),
|
||||
):
|
||||
validate_worker_profiler_config(ProfilerConfig(profiler="cuda"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities", "expected"),
|
||||
[
|
||||
("cuda", None, ("CPU", "CUDA")),
|
||||
("xpu", None, ("CPU", "XPU")),
|
||||
("cpu", None, ("CPU",)),
|
||||
("cuda", ["CUDA"], ("CUDA",)),
|
||||
("xpu", ["XPU"], ("XPU",)),
|
||||
("cuda", ["CPU"], ("CPU",)),
|
||||
("xpu", ["CPU"], ("CPU",)),
|
||||
("cpu", ["CPU"], ("CPU",)),
|
||||
],
|
||||
)
|
||||
def test_worker_creates_platform_torch_profiler(device_type, activities, expected):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
validate_worker_profiler_config(config)
|
||||
profiler = create_worker_profiler(
|
||||
config,
|
||||
worker_name="rank0",
|
||||
local_rank=0,
|
||||
)
|
||||
|
||||
assert profiler is wrapper.return_value
|
||||
wrapper.assert_called_once_with(
|
||||
config,
|
||||
worker_name="rank0",
|
||||
local_rank=0,
|
||||
activities=expected,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker_type", [Worker, XPUWorker])
|
||||
def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type):
|
||||
worker = object.__new__(worker_type)
|
||||
worker.rank = 0
|
||||
worker.local_rank = 0
|
||||
worker.profiler = None
|
||||
worker.profiler_config = ProfilerConfig(
|
||||
profiler="torch", torch_profiler_dir="/tmp/mock"
|
||||
)
|
||||
|
||||
with (
|
||||
patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
worker.profile()
|
||||
worker.profile(is_start=False)
|
||||
worker.profile()
|
||||
|
||||
assert worker.profiler is wrapper.return_value
|
||||
wrapper.assert_called_once()
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
wrapper.return_value.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_immediate_start_stop(default_profiler_config):
|
||||
"""Test standard start without delay."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
@@ -318,13 +533,17 @@ class TestAnnotateProfile:
|
||||
"execute_5_context_1(sq4sk4sqsq16sqsk16)_generation_1(sq1sk11sqsq1sqsk11)"
|
||||
)
|
||||
|
||||
def test_skips_annotation_work_after_profiler_stops(self):
|
||||
def test_skips_annotation_work_when_profiler_does_not_annotate(self):
|
||||
worker = MagicMock()
|
||||
worker.profiler.is_running = False
|
||||
worker.profiler.should_annotate = False
|
||||
|
||||
context = Worker.annotate_profile(worker, scheduler_output=None)
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu_worker.compute_iteration_details"
|
||||
) as compute_iteration_details:
|
||||
context = Worker.annotate_profile(worker, scheduler_output=None)
|
||||
|
||||
worker.profiler.step.assert_called_once_with()
|
||||
compute_iteration_details.assert_not_called()
|
||||
worker.profiler.annotate_context_manager.assert_not_called()
|
||||
assert isinstance(context, nullcontext)
|
||||
|
||||
@@ -837,17 +1056,18 @@ class TestProtonProfilerWrapper:
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_creates_proton_profiler():
|
||||
worker = MagicMock()
|
||||
worker = object.__new__(Worker)
|
||||
worker.rank = 1
|
||||
worker.local_rank = 1
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config = MagicMock(profiler="proton")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
Worker.profile(worker)
|
||||
|
||||
@@ -857,17 +1077,18 @@ def test_gpu_worker_creates_proton_profiler():
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
worker = MagicMock()
|
||||
worker = object.__new__(Worker)
|
||||
worker.rank = 1
|
||||
worker.local_rank = 1
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config = MagicMock(profiler="proton")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
wrapper.return_value.has_cuda_graph_session = False
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
|
||||
+25
-1
@@ -14,6 +14,7 @@ from vllm.utils.hashing import safe_hash
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ProfilerKind = Literal["torch", "cuda", "proton"]
|
||||
TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"]
|
||||
ProtonBackend = Literal["cupti"]
|
||||
ProtonContext = Literal["shadow", "python"]
|
||||
ProtonData = Literal["tree", "trace"]
|
||||
@@ -51,6 +52,12 @@ class ProfilerConfig:
|
||||
worker's traces (CPU & GPU) will be saved under this directory. Note that
|
||||
it must be an absolute path."""
|
||||
|
||||
torch_profiler_activities: list[TorchProfilerActivity] | None = Field(
|
||||
default=None, min_length=1
|
||||
)
|
||||
"""Activities recorded by workers using the torch profiler. When unset,
|
||||
each worker uses its platform default: CPU; CPU and CUDA; or CPU and XPU."""
|
||||
|
||||
proton_profiler_dir: str = ""
|
||||
"""Directory to save Triton Proton profiles. Each worker writes a
|
||||
separate rank-qualified file."""
|
||||
@@ -170,7 +177,24 @@ class ProfilerConfig:
|
||||
@model_validator(mode="after")
|
||||
def _validate_profiler_config(self) -> Self:
|
||||
has_delay_or_limit = self.delay_iterations > 0 or self.max_iterations > 0
|
||||
if self.profiler == "torch" and has_delay_or_limit and not self.ignore_frontend:
|
||||
activities = self.torch_profiler_activities
|
||||
if activities is not None:
|
||||
if self.profiler != "torch":
|
||||
raise ValueError(
|
||||
"torch_profiler_activities is only applicable when profiler "
|
||||
"is set to 'torch'"
|
||||
)
|
||||
if len(activities) != len(set(activities)):
|
||||
raise ValueError(
|
||||
"torch_profiler_activities must not contain duplicates"
|
||||
)
|
||||
records_cpu_activity = activities is None or "CPU" in activities
|
||||
if (
|
||||
self.profiler == "torch"
|
||||
and has_delay_or_limit
|
||||
and not self.ignore_frontend
|
||||
and records_cpu_activity
|
||||
):
|
||||
logger.warning_once(
|
||||
"Using 'torch' profiler with delay_iterations or max_iterations "
|
||||
"while ignore_frontend is False may result in high overhead."
|
||||
|
||||
+106
-11
@@ -6,10 +6,9 @@ import inspect
|
||||
import json
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
|
||||
from glob import glob
|
||||
from typing import Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import torch
|
||||
@@ -18,7 +17,7 @@ from typing_extensions import override
|
||||
|
||||
import vllm.version
|
||||
from vllm.config import ProfilerConfig
|
||||
from vllm.config.profiler import _is_uri_path
|
||||
from vllm.config.profiler import ProfilerKind, TorchProfilerActivity, _is_uri_path
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -57,6 +56,11 @@ class WorkerProfiler(ABC):
|
||||
"""Whether the underlying profiler is currently collecting data."""
|
||||
return self._running
|
||||
|
||||
@property
|
||||
def should_annotate(self) -> bool:
|
||||
"""Whether worker iterations should receive profiler annotations."""
|
||||
return self._running
|
||||
|
||||
@abstractmethod
|
||||
def _start(self) -> None:
|
||||
"""Start the profiler."""
|
||||
@@ -176,7 +180,6 @@ class WorkerProfiler(ABC):
|
||||
return nullcontext()
|
||||
|
||||
|
||||
TorchProfilerActivity = Literal["CPU", "CUDA", "PrivateUse1", "XPU"]
|
||||
TorchProfilerActivityMap = {
|
||||
"CPU": torch.profiler.ProfilerActivity.CPU,
|
||||
"CUDA": torch.profiler.ProfilerActivity.CUDA,
|
||||
@@ -191,7 +194,7 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
profiler_config: ProfilerConfig,
|
||||
worker_name: str,
|
||||
local_rank: int,
|
||||
activities: list[TorchProfilerActivity],
|
||||
activities: Sequence[TorchProfilerActivity],
|
||||
on_trace_ready: Callable[[torch.profiler.profile], None] | None = None,
|
||||
) -> None:
|
||||
super().__init__(profiler_config)
|
||||
@@ -224,7 +227,12 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
use_gzip=profiler_config.torch_profiler_use_gzip,
|
||||
)
|
||||
|
||||
self.dump_cpu_time_total = "CPU" in activities and len(activities) == 1
|
||||
self._records_cpu_activity = "CPU" in activities
|
||||
self.dump_device_time_total = (
|
||||
any(activity != "CPU" for activity in activities)
|
||||
and profiler_config.torch_profiler_dump_cuda_time_total
|
||||
)
|
||||
self.dump_cpu_time_total = self._records_cpu_activity and len(activities) == 1
|
||||
|
||||
# Create profiler schedule if warmup or wait iterations are configured
|
||||
profiler_schedule = None
|
||||
@@ -244,7 +252,7 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
profiler_config.active_iterations,
|
||||
)
|
||||
|
||||
self.profiler = torch.profiler.profile(
|
||||
self._profiler_kwargs = dict(
|
||||
activities=[TorchProfilerActivityMap[activity] for activity in activities],
|
||||
schedule=profiler_schedule,
|
||||
record_shapes=profiler_config.torch_profiler_record_shapes,
|
||||
@@ -253,6 +261,7 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
with_flops=profiler_config.torch_profiler_with_flops,
|
||||
on_trace_ready=trace_handler,
|
||||
)
|
||||
self.profiler: torch.profiler.profile
|
||||
|
||||
# Track if we're using a schedule (need to call step())
|
||||
self._uses_schedule = profiler_schedule is not None
|
||||
@@ -260,10 +269,11 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
# Subtract 1 because profiler.start() already consumes step 0
|
||||
# (WAIT or WARMUP), so only wait + warmup - 1 non-active steps
|
||||
# remain to be advanced through via profiler.step() calls.
|
||||
self._warmup_steps_remaining = max(
|
||||
self._initial_warmup_steps_remaining = max(
|
||||
profiler_config.wait_iterations + profiler_config.warmup_iterations - 1,
|
||||
0,
|
||||
)
|
||||
self._warmup_steps_remaining = self._initial_warmup_steps_remaining
|
||||
self._version_metadata_added = False
|
||||
|
||||
def _build_profiler_table(
|
||||
@@ -319,6 +329,9 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
|
||||
@override
|
||||
def _start(self) -> None:
|
||||
self.profiler = torch.profiler.profile(**self._profiler_kwargs)
|
||||
self._warmup_steps_remaining = self._initial_warmup_steps_remaining
|
||||
self._version_metadata_added = False
|
||||
self.profiler.start()
|
||||
# No-schedule case: Kineto is live immediately. With a schedule this
|
||||
# no-ops and _profiler_step stamps it once WAIT ends.
|
||||
@@ -328,10 +341,9 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
def _stop(self) -> None:
|
||||
self.profiler.stop()
|
||||
|
||||
profiler_config = self.profiler_config
|
||||
rank = self.local_rank
|
||||
if profiler_config.torch_profiler_dump_cuda_time_total:
|
||||
table = self._build_profiler_table(sort_key="self_cuda_time_total")
|
||||
if self.dump_device_time_total:
|
||||
table = self._build_profiler_table(sort_key="self_device_time_total")
|
||||
self._write_profiler_table(rank, table)
|
||||
|
||||
# only print profiler results on rank 0
|
||||
@@ -367,8 +379,15 @@ class TorchProfilerWrapper(WorkerProfiler):
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
@override
|
||||
def should_annotate(self) -> bool:
|
||||
return self._running and self._records_cpu_activity
|
||||
|
||||
@override
|
||||
def annotate_context_manager(self, name: str):
|
||||
if not self.should_annotate:
|
||||
return nullcontext()
|
||||
return torch.profiler.record_function(name)
|
||||
|
||||
|
||||
@@ -606,6 +625,82 @@ class CudaProfilerWrapper(WorkerProfiler):
|
||||
return torch.cuda.nvtx.range(name)
|
||||
|
||||
|
||||
_DEFAULT_TORCH_PROFILER_ACTIVITIES: dict[str, tuple[TorchProfilerActivity, ...]] = {
|
||||
"cpu": ("CPU",),
|
||||
"cuda": ("CPU", "CUDA"),
|
||||
"xpu": ("CPU", "XPU"),
|
||||
}
|
||||
_SUPPORTED_TORCH_PROFILER_ACTIVITIES = {
|
||||
device: frozenset(activities)
|
||||
for device, activities in _DEFAULT_TORCH_PROFILER_ACTIVITIES.items()
|
||||
}
|
||||
_SUPPORTED_PROFILER_KINDS: dict[str, frozenset[ProfilerKind]] = {
|
||||
"cpu": frozenset(("torch",)),
|
||||
"cuda": frozenset(("torch", "cuda", "proton")),
|
||||
"xpu": frozenset(("torch",)),
|
||||
}
|
||||
|
||||
|
||||
def validate_worker_profiler_config(profiler_config: ProfilerConfig) -> None:
|
||||
"""Validate profiler selections against the worker's capabilities."""
|
||||
profiler_type = profiler_config.profiler
|
||||
if profiler_type is None:
|
||||
return
|
||||
device_type = current_platform.device_type
|
||||
if device_type not in _SUPPORTED_PROFILER_KINDS:
|
||||
raise ValueError(f"Unsupported profiler device type: {device_type}")
|
||||
supported_kinds = _SUPPORTED_PROFILER_KINDS[device_type]
|
||||
if profiler_type not in supported_kinds:
|
||||
supported_names = ", ".join(sorted(supported_kinds))
|
||||
raise ValueError(
|
||||
f"Unsupported profiler type for {device_type}: "
|
||||
f"{profiler_type}. Supported profiler types: {supported_names}."
|
||||
)
|
||||
if profiler_type == "torch":
|
||||
default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[device_type]
|
||||
supported_activities = _SUPPORTED_TORCH_PROFILER_ACTIVITIES[device_type]
|
||||
configured = profiler_config.torch_profiler_activities
|
||||
activities = default_activities if configured is None else configured
|
||||
unsupported = set(activities) - supported_activities
|
||||
if unsupported:
|
||||
unsupported_names = ", ".join(sorted(unsupported))
|
||||
supported_names = ", ".join(sorted(supported_activities))
|
||||
raise ValueError(
|
||||
f"Unsupported torch profiler activities for "
|
||||
f"{device_type}: {unsupported_names}. "
|
||||
f"Supported activities: {supported_names}."
|
||||
)
|
||||
|
||||
|
||||
def create_worker_profiler(
|
||||
profiler_config: ProfilerConfig,
|
||||
*,
|
||||
worker_name: str,
|
||||
local_rank: int,
|
||||
) -> WorkerProfiler:
|
||||
"""Create a profiler using a validated config and platform defaults."""
|
||||
profiler_type = profiler_config.profiler
|
||||
if profiler_type == "torch":
|
||||
default_activities = _DEFAULT_TORCH_PROFILER_ACTIVITIES[
|
||||
current_platform.device_type
|
||||
]
|
||||
configured = profiler_config.torch_profiler_activities
|
||||
logger.debug("Starting torch profiler with trace name: %s", worker_name)
|
||||
return TorchProfilerWrapper(
|
||||
profiler_config,
|
||||
worker_name=worker_name,
|
||||
local_rank=local_rank,
|
||||
activities=default_activities if configured is None else tuple(configured),
|
||||
)
|
||||
if profiler_type == "cuda":
|
||||
logger.debug("Starting CUDA profiler")
|
||||
return CudaProfilerWrapper(profiler_config)
|
||||
|
||||
assert profiler_type == "proton", f"Unknown profiler type: {profiler_type}"
|
||||
logger.debug("Starting Proton profiler with trace name: %s", worker_name)
|
||||
return ProtonProfilerWrapper(profiler_config, worker_name=worker_name)
|
||||
|
||||
|
||||
def create_graph_capture_profiler(
|
||||
profiler_config: ProfilerConfig, global_rank: int
|
||||
) -> WorkerProfiler | None:
|
||||
|
||||
@@ -208,9 +208,11 @@ class AsyncLLM(EngineClient):
|
||||
pass
|
||||
|
||||
self.profiler = profiler
|
||||
configured_activities = vllm_config.profiler_config.torch_profiler_activities
|
||||
if (
|
||||
vllm_config.profiler_config.profiler == "torch"
|
||||
and not vllm_config.profiler_config.ignore_frontend
|
||||
and (configured_activities is None or "CPU" in configured_activities)
|
||||
):
|
||||
profiler_dir = vllm_config.profiler_config.torch_profiler_dir
|
||||
logger.info(
|
||||
|
||||
@@ -7,7 +7,6 @@ import vllm.v1.worker.cpu.shm # noqa # isort: skip
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
@@ -16,7 +15,6 @@ from vllm import envs
|
||||
from vllm.config import CompilationMode, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.profiler.wrapper import TorchProfilerWrapper
|
||||
from vllm.utils.cpu_resource_utils import (
|
||||
get_allowed_cpu_list,
|
||||
get_memory_node_info,
|
||||
@@ -106,18 +104,6 @@ class CPUWorker(Worker):
|
||||
|
||||
self.parallel_config.disable_custom_all_reduce = True
|
||||
|
||||
# Torch profiler. Enabled and configured through profiler_config.
|
||||
self.profiler: Any | None = None
|
||||
profiler_config = vllm_config.profiler_config
|
||||
if profiler_config.profiler == "torch":
|
||||
worker_name = f"{vllm_config.instance_id}-rank-{self.rank}"
|
||||
self.profiler = TorchProfilerWrapper(
|
||||
profiler_config,
|
||||
worker_name=worker_name,
|
||||
local_rank=self.local_rank,
|
||||
activities=["CPU"],
|
||||
)
|
||||
|
||||
def init_device(self):
|
||||
self.device = torch.device("cpu")
|
||||
|
||||
@@ -287,11 +273,3 @@ class CPUWorker(Worker):
|
||||
language_model=self.compilation_config.compilation_time,
|
||||
encoder=self.compilation_config.encoder_compilation_time,
|
||||
)
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
if self.profiler is None:
|
||||
raise RuntimeError("Profiler is not enabled.")
|
||||
if is_start:
|
||||
self.profiler.start()
|
||||
else:
|
||||
self.profiler.stop()
|
||||
|
||||
@@ -61,10 +61,9 @@ from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import (
|
||||
CudaProfilerWrapper,
|
||||
ProtonProfilerWrapper,
|
||||
TorchProfilerWrapper,
|
||||
create_graph_capture_profiler,
|
||||
create_worker_profiler,
|
||||
validate_worker_profiler_config,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import SupportedTask
|
||||
@@ -222,6 +221,7 @@ class Worker(WorkerBase):
|
||||
# so we have all the information needed for proper trace naming.
|
||||
self.profiler: Any | None = None
|
||||
self.profiler_config = vllm_config.profiler_config
|
||||
validate_worker_profiler_config(self.profiler_config)
|
||||
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
|
||||
@@ -1038,7 +1038,7 @@ class Worker(WorkerBase):
|
||||
return nullcontext()
|
||||
|
||||
self.profiler.step()
|
||||
if not self.profiler.is_running:
|
||||
if not self.profiler.should_annotate:
|
||||
return nullcontext()
|
||||
|
||||
iteration_details = compute_iteration_details(scheduler_output)
|
||||
@@ -1262,7 +1262,6 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
if is_start:
|
||||
profiler_type = self.profiler_config.profiler
|
||||
# Generate the trace name by combining prefix with comprehensive rank suffix
|
||||
from vllm.distributed.utils import get_worker_rank_suffix
|
||||
|
||||
@@ -1274,36 +1273,16 @@ class Worker(WorkerBase):
|
||||
else:
|
||||
trace_name = rank_suffix
|
||||
|
||||
if profiler_type == "proton" and self.profiler is not None:
|
||||
if self.profiler_config.profiler == "proton" and self.profiler is not None:
|
||||
self.profiler.set_output_name(trace_name)
|
||||
|
||||
# Create the profiler wrapper only on the first start call
|
||||
if self.profiler is None:
|
||||
if profiler_type == "torch":
|
||||
self.profiler = TorchProfilerWrapper(
|
||||
self.profiler_config,
|
||||
worker_name=trace_name,
|
||||
local_rank=self.local_rank,
|
||||
activities=["CPU", "CUDA"],
|
||||
)
|
||||
logger.debug(
|
||||
"Starting torch profiler with trace name: %s", trace_name
|
||||
)
|
||||
elif profiler_type == "cuda":
|
||||
self.profiler = CudaProfilerWrapper(self.profiler_config)
|
||||
logger.debug("Starting CUDA profiler")
|
||||
elif profiler_type == "proton":
|
||||
self.profiler = ProtonProfilerWrapper(
|
||||
self.profiler_config, worker_name=trace_name
|
||||
)
|
||||
logger.debug(
|
||||
"Starting Proton profiler with trace name: %s", trace_name
|
||||
)
|
||||
else:
|
||||
# Config validation should prevent this code being reached
|
||||
raise ValueError(
|
||||
f"Invalid profiler value of {self.profiler_config.profiler}"
|
||||
)
|
||||
self.profiler = create_worker_profiler(
|
||||
self.profiler_config,
|
||||
worker_name=trace_name,
|
||||
local_rank=self.local_rank,
|
||||
)
|
||||
|
||||
self.profiler.start()
|
||||
else:
|
||||
|
||||
@@ -8,7 +8,6 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import TorchProfilerWrapper
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.utils import report_usage_stats
|
||||
@@ -225,33 +224,6 @@ class XPUWorker(Worker):
|
||||
# If usage stat is enabled, collect relevant info.
|
||||
report_usage_stats(self.vllm_config)
|
||||
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
if self.profiler_config is None or self.profiler_config.profiler is None:
|
||||
raise RuntimeError(
|
||||
"Profiling is not enabled. Please set --profiler-config to enable "
|
||||
"profiling. Example: "
|
||||
"'--profiler-config.profiler=torch --profiler-config.torch_profiler_dir"
|
||||
"=YOUR_DIR_PATH_TO_DUMP_TRACE'"
|
||||
)
|
||||
|
||||
if is_start and self.profiler is None:
|
||||
from vllm.distributed.utils import get_worker_rank_suffix
|
||||
|
||||
rank_suffix = get_worker_rank_suffix(global_rank=self.rank)
|
||||
trace_name = (
|
||||
f"{profile_prefix}_{rank_suffix}" if profile_prefix else rank_suffix
|
||||
)
|
||||
|
||||
self.profiler = TorchProfilerWrapper(
|
||||
self.profiler_config,
|
||||
worker_name=trace_name,
|
||||
local_rank=self.local_rank,
|
||||
activities=["CPU", "XPU"],
|
||||
)
|
||||
logger.debug("Starting torch profiler with trace name: %s", trace_name)
|
||||
|
||||
super().profile(is_start=is_start, profile_prefix=profile_prefix)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
logger.info(
|
||||
"XPUWorker shutdown: cleaning up (rank=%d, local_rank=%d)",
|
||||
|
||||
Reference in New Issue
Block a user