[Profiler] Unify platform-aware torch profiling (#57460)
This commit is contained in:
@@ -60,13 +60,7 @@ VISION_PROMPT = {
|
||||
}
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
|
||||
def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
|
||||
renderer = MagicMock()
|
||||
engine_core = MagicMock()
|
||||
engine_core.profile_async = AsyncMock()
|
||||
@@ -90,6 +84,16 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
"make_async_mp_client",
|
||||
MagicMock(return_value=engine_core),
|
||||
)
|
||||
return engine_core
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
@@ -103,6 +107,25 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
|
||||
|
||||
|
||||
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config.profiler_config.profiler = "torch"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
assert engine.profiler is None
|
||||
profiler.assert_not_called()
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncLLM,
|
||||
request_id: str,
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock, call, patch
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config import (
|
||||
@@ -17,10 +18,17 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.config.profiler import _is_uri_path
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import ProtonProfilerWrapper, WorkerProfiler
|
||||
from vllm.profiler.wrapper import (
|
||||
ProtonProfilerWrapper,
|
||||
TorchProfilerWrapper,
|
||||
WorkerProfiler,
|
||||
create_worker_profiler,
|
||||
validate_worker_profiler_config,
|
||||
)
|
||||
from vllm.v1.core.sched.output import CachedRequestData
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
from vllm.v1.worker.xpu_worker import XPUWorker
|
||||
|
||||
|
||||
class ConcreteWorkerProfiler(WorkerProfiler):
|
||||
@@ -51,6 +59,213 @@ def default_profiler_config():
|
||||
)
|
||||
|
||||
|
||||
def test_torch_profiler_rebuilds_one_shot_profiler_each_round(tmp_path):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CUDA"],
|
||||
torch_profiler_dump_cuda_time_total=False,
|
||||
warmup_iterations=2,
|
||||
)
|
||||
profilers = [MagicMock(), MagicMock()]
|
||||
for profiler in profilers:
|
||||
profiler.profiler = MagicMock()
|
||||
|
||||
with patch(
|
||||
"vllm.profiler.wrapper.torch.profiler.profile", side_effect=profilers
|
||||
) as profile:
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="worker", local_rank=0, activities=["CUDA"]
|
||||
)
|
||||
profile.assert_not_called()
|
||||
|
||||
wrapper.start()
|
||||
assert not wrapper.should_annotate
|
||||
assert not wrapper._profiler_step()
|
||||
wrapper.stop()
|
||||
wrapper.start()
|
||||
assert not wrapper._profiler_step()
|
||||
wrapper.stop()
|
||||
|
||||
assert profile.call_count == 2
|
||||
assert profile.call_args.kwargs["activities"] == [
|
||||
torch.profiler.ProfilerActivity.CUDA
|
||||
]
|
||||
for profiler in profilers:
|
||||
profiler.start.assert_called_once_with()
|
||||
profiler.step.assert_called_once_with()
|
||||
profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_torch_profiler_records_each_profile_round(tmp_path):
|
||||
traces: list[torch.profiler.profile] = []
|
||||
wrapper = TorchProfilerWrapper(
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_dump_cuda_time_total=False,
|
||||
),
|
||||
worker_name="worker",
|
||||
local_rank=1,
|
||||
activities=["CPU"],
|
||||
on_trace_ready=traces.append,
|
||||
)
|
||||
|
||||
for run in range(2):
|
||||
wrapper.start()
|
||||
with torch.profiler.record_function(f"run_{run}"):
|
||||
pass
|
||||
wrapper.stop()
|
||||
|
||||
assert len(traces) == 2
|
||||
for run, trace in enumerate(traces):
|
||||
assert {
|
||||
event.name for event in trace.events() if event.name.startswith("run_")
|
||||
} == {f"run_{run}"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities", [["CPU", "CUDA"], ["CUDA"], ["CPU", "XPU"], ["XPU"]]
|
||||
)
|
||||
@pytest.mark.parametrize("dump_device_time", [True, False])
|
||||
def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device_time):
|
||||
"""Device summaries honor the dump option on both CUDA and XPU."""
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_dump_cuda_time_total=dump_device_time,
|
||||
)
|
||||
with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile:
|
||||
profile.return_value.key_averages.return_value.table.return_value = (
|
||||
"device times"
|
||||
)
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="worker", local_rank=0, activities=activities
|
||||
)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
summary = tmp_path / "profiler_out_0.txt"
|
||||
assert summary.exists() == dump_device_time
|
||||
assert ("device times" in capsys.readouterr().out) == dump_device_time
|
||||
if dump_device_time:
|
||||
assert summary.read_text() == "device times\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities",
|
||||
[[], ["CPU", "CPU"], ["INVALID"]],
|
||||
)
|
||||
def test_torch_profiler_activities_reject_invalid_values(activities):
|
||||
with pytest.raises(ValueError, match="torch_profiler_activities"):
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
|
||||
def test_torch_profiler_activities_require_torch_profiler():
|
||||
with pytest.raises(ValueError, match="only applicable"):
|
||||
ProfilerConfig(torch_profiler_activities=["CPU"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities"),
|
||||
[
|
||||
("cuda", ["XPU"]),
|
||||
("xpu", ["CUDA"]),
|
||||
("cpu", ["CUDA"]),
|
||||
("cpu", ["XPU"]),
|
||||
],
|
||||
)
|
||||
def test_worker_rejects_unsupported_activities_at_startup(device_type, activities):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
pytest.raises(ValueError, match="Unsupported torch profiler activities"),
|
||||
):
|
||||
validate_worker_profiler_config(config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_type", ["cpu", "xpu"])
|
||||
def test_worker_rejects_cuda_profiler_on_other_devices(device_type):
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
pytest.raises(ValueError, match="Unsupported profiler type"),
|
||||
):
|
||||
validate_worker_profiler_config(ProfilerConfig(profiler="cuda"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities", "expected"),
|
||||
[
|
||||
("cuda", None, ("CPU", "CUDA")),
|
||||
("xpu", None, ("CPU", "XPU")),
|
||||
("cpu", None, ("CPU",)),
|
||||
("cuda", ["CUDA"], ("CUDA",)),
|
||||
("xpu", ["XPU"], ("XPU",)),
|
||||
("cuda", ["CPU"], ("CPU",)),
|
||||
("xpu", ["CPU"], ("CPU",)),
|
||||
("cpu", ["CPU"], ("CPU",)),
|
||||
],
|
||||
)
|
||||
def test_worker_creates_platform_torch_profiler(device_type, activities, expected):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
torch_profiler_activities=activities,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "device_type", device_type),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
validate_worker_profiler_config(config)
|
||||
profiler = create_worker_profiler(
|
||||
config,
|
||||
worker_name="rank0",
|
||||
local_rank=0,
|
||||
)
|
||||
|
||||
assert profiler is wrapper.return_value
|
||||
wrapper.assert_called_once_with(
|
||||
config,
|
||||
worker_name="rank0",
|
||||
local_rank=0,
|
||||
activities=expected,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("worker_type", [Worker, XPUWorker])
|
||||
def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type):
|
||||
worker = object.__new__(worker_type)
|
||||
worker.rank = 0
|
||||
worker.local_rank = 0
|
||||
worker.profiler = None
|
||||
worker.profiler_config = ProfilerConfig(
|
||||
profiler="torch", torch_profiler_dir="/tmp/mock"
|
||||
)
|
||||
|
||||
with (
|
||||
patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
worker.profile()
|
||||
worker.profile(is_start=False)
|
||||
worker.profile()
|
||||
|
||||
assert worker.profiler is wrapper.return_value
|
||||
wrapper.assert_called_once()
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
wrapper.return_value.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_immediate_start_stop(default_profiler_config):
|
||||
"""Test standard start without delay."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
@@ -318,13 +533,17 @@ class TestAnnotateProfile:
|
||||
"execute_5_context_1(sq4sk4sqsq16sqsk16)_generation_1(sq1sk11sqsq1sqsk11)"
|
||||
)
|
||||
|
||||
def test_skips_annotation_work_after_profiler_stops(self):
|
||||
def test_skips_annotation_work_when_profiler_does_not_annotate(self):
|
||||
worker = MagicMock()
|
||||
worker.profiler.is_running = False
|
||||
worker.profiler.should_annotate = False
|
||||
|
||||
context = Worker.annotate_profile(worker, scheduler_output=None)
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu_worker.compute_iteration_details"
|
||||
) as compute_iteration_details:
|
||||
context = Worker.annotate_profile(worker, scheduler_output=None)
|
||||
|
||||
worker.profiler.step.assert_called_once_with()
|
||||
compute_iteration_details.assert_not_called()
|
||||
worker.profiler.annotate_context_manager.assert_not_called()
|
||||
assert isinstance(context, nullcontext)
|
||||
|
||||
@@ -837,17 +1056,18 @@ class TestProtonProfilerWrapper:
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_creates_proton_profiler():
|
||||
worker = MagicMock()
|
||||
worker = object.__new__(Worker)
|
||||
worker.rank = 1
|
||||
worker.local_rank = 1
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config = MagicMock(profiler="proton")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
Worker.profile(worker)
|
||||
|
||||
@@ -857,17 +1077,18 @@ def test_gpu_worker_creates_proton_profiler():
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
def test_gpu_worker_recreates_proton_profiler_for_each_run():
|
||||
worker = MagicMock()
|
||||
worker = object.__new__(Worker)
|
||||
worker.rank = 1
|
||||
worker.local_rank = 1
|
||||
worker.profiler = None
|
||||
worker.profiler_config.profiler = "proton"
|
||||
worker.profiler_config = MagicMock(profiler="proton")
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.utils.get_worker_rank_suffix",
|
||||
return_value="rank1",
|
||||
),
|
||||
patch("vllm.v1.worker.gpu_worker.ProtonProfilerWrapper") as wrapper,
|
||||
patch("vllm.profiler.wrapper.ProtonProfilerWrapper") as wrapper,
|
||||
):
|
||||
wrapper.return_value.has_cuda_graph_session = False
|
||||
Worker.profile(worker, profile_prefix="first")
|
||||
|
||||
Reference in New Issue
Block a user