[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)

Signed-off-by: czhu <conway.zhu@cohere.com>
Signed-off-by: Conway Zhu <conway.zhu@cohere.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
czhu-cohere
2026-10-06 05:21:19 +00:00
committed by GitHub
co-authored by aoshen02
parent 9a1f6fd4cb
commit 6fddc9b21e
25 changed files with 1497 additions and 164 deletions
+283 -9
View File
@@ -2,8 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import threading
import time
from contextlib import ExitStack
from http import HTTPStatus
from unittest.mock import AsyncMock, MagicMock, call
import pytest
@@ -11,7 +13,7 @@ import pytest
import vllm.v1.engine.async_llm as async_llm_module
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.config import LoggingConfig, VllmConfig
from vllm.config import LoggingConfig, ProfilerConfig, VllmConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
@@ -20,10 +22,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.exceptions import VLLMValidationError
from vllm.exceptions import ProfilerAlreadyActiveError, VLLMValidationError
from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.profiler.wrapper import WorkerProfiler
from vllm.sampling_params import RequestOutputKind
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.engine.async_llm import AsyncLLM
@@ -87,13 +90,19 @@ def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
return engine_core
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
def _mock_vllm_config() -> MagicMock:
vllm_config = MagicMock()
vllm_config.logging_config = LoggingConfig()
vllm_config.observability_config.otlp_traces_endpoint = None
vllm_config.scheduler_config.stream_interval = 1
return vllm_config
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
vllm_config = _mock_vllm_config()
vllm_config.profiler_config.profiler = "cuda"
vllm_config.profiler_config.ignore_frontend = False
vllm_config.profiler_config.should_profile_frontend = False
engine_core = _mock_async_llm_dependencies(monkeypatch)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
@@ -105,22 +114,22 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
asyncio.run(profile())
assert engine.profiler is None
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
engine_core.profile_async.assert_has_awaits(
[call(True, None, None, None), call(False)]
)
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
monkeypatch: pytest.MonkeyPatch,
):
vllm_config = MagicMock()
vllm_config.logging_config = LoggingConfig()
vllm_config.observability_config.otlp_traces_endpoint = None
vllm_config.scheduler_config.stream_interval = 1
vllm_config = _mock_vllm_config()
vllm_config.profiler_config.profiler = "torch"
vllm_config.profiler_config.ignore_frontend = False
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
vllm_config.profiler_config.should_profile_frontend = False
_mock_async_llm_dependencies(monkeypatch)
profiler = MagicMock()
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
@@ -128,6 +137,271 @@ def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
profiler.assert_not_called()
def test_profile_forwards_overrides_rejects_duplicate_and_allows_restart(
monkeypatch: pytest.MonkeyPatch,
):
vllm_config = _mock_vllm_config()
vllm_config.profiler_config.should_profile_frontend = False
engine_core = _mock_async_llm_dependencies(monkeypatch)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
async def profile():
for kwargs in (
{"profile_prefix": "../trace"},
{"delay_iterations": -1},
{"max_iterations": -1},
):
with pytest.raises(ValueError):
await engine.start_profile(**kwargs)
await engine.start_profile(
"session",
delay_iterations=5,
max_iterations=2,
)
with pytest.raises(ProfilerAlreadyActiveError) as exc_info:
await engine.start_profile("duplicate")
assert exc_info.value.http_status == HTTPStatus.CONFLICT
await engine.stop_profile()
await engine.start_profile("next-session")
await engine.stop_profile()
asyncio.run(profile())
engine_core.profile_async.assert_has_awaits(
[
call(True, "session", 5, 2),
call(False),
call(True, "next-session", None, None),
call(False),
]
)
def test_stop_profile_waits_for_inflight_start(monkeypatch: pytest.MonkeyPatch):
vllm_config = _mock_vllm_config()
engine_core = _mock_async_llm_dependencies(monkeypatch)
events: list[str] = []
frontend_profiler = MagicMock()
frontend_profiler.start.side_effect = lambda: events.append("frontend_start")
frontend_profiler.stop.side_effect = lambda: events.append("frontend_stop")
engine = AsyncLLM(
vllm_config,
MagicMock(),
log_stats=False,
profiler=frontend_profiler,
)
async def profile():
start_rpc_entered = asyncio.Event()
release_start_rpc = asyncio.Event()
async def profile_async(is_start, *args):
if is_start:
start_rpc_entered.set()
await release_start_rpc.wait()
events.append("engine_start_done")
else:
events.append("engine_stop")
engine_core.profile_async.side_effect = profile_async
start_task = asyncio.create_task(engine.start_profile())
await start_rpc_entered.wait()
stop_task = asyncio.create_task(engine.stop_profile())
await asyncio.sleep(0)
release_start_rpc.set()
await asyncio.wait_for(asyncio.gather(start_task, stop_task), timeout=5)
asyncio.run(profile())
assert events[:2] == ["engine_start_done", "frontend_start"]
assert sorted(events[2:]) == ["engine_stop", "frontend_stop"]
assert engine._profile_session_active is False
@pytest.mark.parametrize("failure", ["error", "cancel"])
def test_interrupted_engine_profile_start_skips_frontend_profiler(
monkeypatch: pytest.MonkeyPatch, failure: str
):
"""The frontend trace starts only after the engine start succeeds. A
cancelled start may still reach the engine, so it keeps the session active
until /stop_profile."""
vllm_config = _mock_vllm_config()
engine_core = _mock_async_llm_dependencies(monkeypatch)
frontend_profiler = MagicMock()
engine = AsyncLLM(
vllm_config,
MagicMock(),
log_stats=False,
profiler=frontend_profiler,
)
async def profile():
start_rpc_entered = asyncio.Event()
async def profile_async(is_start, *args):
if not is_start:
return
start_rpc_entered.set()
if failure == "error":
raise RuntimeError("engine profiler failed to start")
await asyncio.Event().wait()
engine_core.profile_async.side_effect = profile_async
start_task = asyncio.create_task(engine.start_profile())
await start_rpc_entered.wait()
if failure == "cancel":
start_task.cancel()
expected = RuntimeError if failure == "error" else asyncio.CancelledError
with pytest.raises(expected):
await start_task
assert engine._profile_session_active is (failure == "cancel")
await engine.stop_profile()
assert engine._profile_session_active is False
asyncio.run(profile())
frontend_profiler.start.assert_not_called()
class _BlockingStartProfiler(WorkerProfiler):
def __init__(self) -> None:
self.in_start = threading.Event()
self.release_start = threading.Event()
self.stop_call_count = 0
super().__init__(
ProfilerConfig(profiler="torch", torch_profiler_dir="/tmp/mock")
)
def _start(self) -> None:
self.in_start.set()
if not self.release_start.wait(timeout=5):
raise TimeoutError("start was never released")
def _stop(self) -> None:
self.stop_call_count += 1
def test_stop_after_cancelled_frontend_start_stops_frontend_profiler(
monkeypatch: pytest.MonkeyPatch,
):
"""Cancelling start_profile while the frontend profiler is starting keeps
the session active, and the next stop_profile still stops the backend."""
vllm_config = _mock_vllm_config()
engine_core = _mock_async_llm_dependencies(monkeypatch)
frontend_profiler = _BlockingStartProfiler()
engine = AsyncLLM(
vllm_config,
MagicMock(),
log_stats=False,
profiler=frontend_profiler,
)
async def profile():
try:
start_task = asyncio.create_task(engine.start_profile())
assert await asyncio.to_thread(frontend_profiler.in_start.wait, 5)
start_task.cancel()
with pytest.raises(asyncio.CancelledError):
await start_task
assert engine._profile_session_active is True
stop_task = asyncio.create_task(engine.stop_profile())
done, _ = await asyncio.wait({stop_task}, timeout=0.1)
assert not done, "stop_profile did not wait for the in-flight start"
frontend_profiler.release_start.set()
await asyncio.wait_for(stop_task, timeout=5)
finally:
frontend_profiler.release_start.set()
asyncio.run(profile())
assert frontend_profiler.stop_call_count == 1
assert frontend_profiler._active is False
assert frontend_profiler._running is False
assert engine._profile_session_active is False
assert engine_core.profile_async.await_args_list == [
call(True, None, None, None),
call(False),
]
def test_multi_client_profile_uses_idempotent_engine_requests(
monkeypatch: pytest.MonkeyPatch, tmp_path
):
vllm_config = _mock_vllm_config()
vllm_config.profiler_config = ProfilerConfig(
profiler="torch",
torch_profiler_dir=str(tmp_path),
torch_profiler_activities=["CPU"],
)
engine_core = _mock_async_llm_dependencies(monkeypatch)
profiler = MagicMock()
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
engines = [
AsyncLLM(
vllm_config,
MagicMock(),
log_stats=False,
client_count=2,
client_index=client_index,
)
for client_index in range(2)
]
assert all(engine.profiler is None for engine in engines)
profiler.assert_not_called()
async def profile():
await engines[0].start_profile()
await engines[1].start_profile()
await engines[1].stop_profile()
await engines[0].start_profile()
asyncio.run(profile())
engine_core.profile_async.assert_has_awaits(
[
call(True, None, None, None),
call(True, None, None, None),
call(False),
call(True, None, None, None),
]
)
def test_frontend_profiler_ignores_worker_iteration_bounds(
monkeypatch: pytest.MonkeyPatch, tmp_path
):
vllm_config = _mock_vllm_config()
vllm_config.profiler_config = ProfilerConfig(
profiler="torch",
torch_profiler_dir=str(tmp_path),
torch_profiler_activities=["CPU", "CUDA"],
delay_iterations=5,
max_iterations=2,
wait_iterations=3,
warmup_iterations=1,
)
_mock_async_llm_dependencies(monkeypatch)
profiler = MagicMock()
monkeypatch.setattr("vllm.profiler.wrapper.TorchProfilerWrapper", profiler)
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
frontend_config = profiler.call_args.args[0]
assert frontend_config.delay_iterations == 0
assert frontend_config.max_iterations == 0
assert frontend_config.wait_iterations == 0
assert frontend_config.warmup_iterations == 0
assert engine.profiler is profiler.return_value
async def generate(
engine: AsyncLLM,
request_id: str,