[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)
Signed-off-by: czhu <conway.zhu@cohere.com> Signed-off-by: Conway Zhu <conway.zhu@cohere.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from contextlib import ExitStack
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, MagicMock, call
|
||||
|
||||
import pytest
|
||||
@@ -11,7 +13,7 @@ import pytest
|
||||
import vllm.v1.engine.async_llm as async_llm_module
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.config import LoggingConfig, VllmConfig
|
||||
from vllm.config import LoggingConfig, ProfilerConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
@@ -20,10 +22,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import ProfilerAlreadyActiveError, VLLMValidationError
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import WorkerProfiler
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -87,13 +90,19 @@ def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
|
||||
return engine_core
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
def _mock_vllm_config() -> MagicMock:
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.logging_config = LoggingConfig()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
return vllm_config
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
@@ -105,22 +114,22 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
asyncio.run(profile())
|
||||
|
||||
assert engine.profiler is None
|
||||
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[call(True, None, None, None), call(False)]
|
||||
)
|
||||
|
||||
|
||||
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.logging_config = LoggingConfig()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.profiler = "torch"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
|
||||
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
@@ -128,6 +137,271 @@ def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
profiler.assert_not_called()
|
||||
|
||||
|
||||
def test_profile_forwards_overrides_rejects_duplicate_and_allows_restart(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
async def profile():
|
||||
for kwargs in (
|
||||
{"profile_prefix": "../trace"},
|
||||
{"delay_iterations": -1},
|
||||
{"max_iterations": -1},
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
await engine.start_profile(**kwargs)
|
||||
await engine.start_profile(
|
||||
"session",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
with pytest.raises(ProfilerAlreadyActiveError) as exc_info:
|
||||
await engine.start_profile("duplicate")
|
||||
assert exc_info.value.http_status == HTTPStatus.CONFLICT
|
||||
await engine.stop_profile()
|
||||
await engine.start_profile("next-session")
|
||||
await engine.stop_profile()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[
|
||||
call(True, "session", 5, 2),
|
||||
call(False),
|
||||
call(True, "next-session", None, None),
|
||||
call(False),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_stop_profile_waits_for_inflight_start(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
events: list[str] = []
|
||||
frontend_profiler = MagicMock()
|
||||
frontend_profiler.start.side_effect = lambda: events.append("frontend_start")
|
||||
frontend_profiler.stop.side_effect = lambda: events.append("frontend_stop")
|
||||
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
start_rpc_entered = asyncio.Event()
|
||||
release_start_rpc = asyncio.Event()
|
||||
|
||||
async def profile_async(is_start, *args):
|
||||
if is_start:
|
||||
start_rpc_entered.set()
|
||||
await release_start_rpc.wait()
|
||||
events.append("engine_start_done")
|
||||
else:
|
||||
events.append("engine_stop")
|
||||
|
||||
engine_core.profile_async.side_effect = profile_async
|
||||
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
await start_rpc_entered.wait()
|
||||
|
||||
stop_task = asyncio.create_task(engine.stop_profile())
|
||||
await asyncio.sleep(0)
|
||||
release_start_rpc.set()
|
||||
await asyncio.wait_for(asyncio.gather(start_task, stop_task), timeout=5)
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
assert events[:2] == ["engine_start_done", "frontend_start"]
|
||||
assert sorted(events[2:]) == ["engine_stop", "frontend_stop"]
|
||||
assert engine._profile_session_active is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["error", "cancel"])
|
||||
def test_interrupted_engine_profile_start_skips_frontend_profiler(
|
||||
monkeypatch: pytest.MonkeyPatch, failure: str
|
||||
):
|
||||
"""The frontend trace starts only after the engine start succeeds. A
|
||||
cancelled start may still reach the engine, so it keeps the session active
|
||||
until /stop_profile."""
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
frontend_profiler = MagicMock()
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
start_rpc_entered = asyncio.Event()
|
||||
|
||||
async def profile_async(is_start, *args):
|
||||
if not is_start:
|
||||
return
|
||||
start_rpc_entered.set()
|
||||
if failure == "error":
|
||||
raise RuntimeError("engine profiler failed to start")
|
||||
await asyncio.Event().wait()
|
||||
|
||||
engine_core.profile_async.side_effect = profile_async
|
||||
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
await start_rpc_entered.wait()
|
||||
if failure == "cancel":
|
||||
start_task.cancel()
|
||||
expected = RuntimeError if failure == "error" else asyncio.CancelledError
|
||||
with pytest.raises(expected):
|
||||
await start_task
|
||||
|
||||
assert engine._profile_session_active is (failure == "cancel")
|
||||
await engine.stop_profile()
|
||||
assert engine._profile_session_active is False
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
frontend_profiler.start.assert_not_called()
|
||||
|
||||
|
||||
class _BlockingStartProfiler(WorkerProfiler):
|
||||
def __init__(self) -> None:
|
||||
self.in_start = threading.Event()
|
||||
self.release_start = threading.Event()
|
||||
self.stop_call_count = 0
|
||||
super().__init__(
|
||||
ProfilerConfig(profiler="torch", torch_profiler_dir="/tmp/mock")
|
||||
)
|
||||
|
||||
def _start(self) -> None:
|
||||
self.in_start.set()
|
||||
if not self.release_start.wait(timeout=5):
|
||||
raise TimeoutError("start was never released")
|
||||
|
||||
def _stop(self) -> None:
|
||||
self.stop_call_count += 1
|
||||
|
||||
|
||||
def test_stop_after_cancelled_frontend_start_stops_frontend_profiler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Cancelling start_profile while the frontend profiler is starting keeps
|
||||
the session active, and the next stop_profile still stops the backend."""
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
frontend_profiler = _BlockingStartProfiler()
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
try:
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
assert await asyncio.to_thread(frontend_profiler.in_start.wait, 5)
|
||||
start_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await start_task
|
||||
assert engine._profile_session_active is True
|
||||
|
||||
stop_task = asyncio.create_task(engine.stop_profile())
|
||||
done, _ = await asyncio.wait({stop_task}, timeout=0.1)
|
||||
assert not done, "stop_profile did not wait for the in-flight start"
|
||||
frontend_profiler.release_start.set()
|
||||
await asyncio.wait_for(stop_task, timeout=5)
|
||||
finally:
|
||||
frontend_profiler.release_start.set()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
assert frontend_profiler.stop_call_count == 1
|
||||
assert frontend_profiler._active is False
|
||||
assert frontend_profiler._running is False
|
||||
assert engine._profile_session_active is False
|
||||
assert engine_core.profile_async.await_args_list == [
|
||||
call(True, None, None, None),
|
||||
call(False),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_client_profile_uses_idempotent_engine_requests(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CPU"],
|
||||
)
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
|
||||
engines = [
|
||||
AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
client_count=2,
|
||||
client_index=client_index,
|
||||
)
|
||||
for client_index in range(2)
|
||||
]
|
||||
|
||||
assert all(engine.profiler is None for engine in engines)
|
||||
profiler.assert_not_called()
|
||||
|
||||
async def profile():
|
||||
await engines[0].start_profile()
|
||||
await engines[1].start_profile()
|
||||
await engines[1].stop_profile()
|
||||
await engines[0].start_profile()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[
|
||||
call(True, None, None, None),
|
||||
call(True, None, None, None),
|
||||
call(False),
|
||||
call(True, None, None, None),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_frontend_profiler_ignores_worker_iteration_bounds(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CPU", "CUDA"],
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
wait_iterations=3,
|
||||
warmup_iterations=1,
|
||||
)
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr("vllm.profiler.wrapper.TorchProfilerWrapper", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
frontend_config = profiler.call_args.args[0]
|
||||
assert frontend_config.delay_iterations == 0
|
||||
assert frontend_config.max_iterations == 0
|
||||
assert frontend_config.wait_iterations == 0
|
||||
assert frontend_config.warmup_iterations == 0
|
||||
assert engine.profiler is profiler.return_value
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncLLM,
|
||||
request_id: str,
|
||||
|
||||
Reference in New Issue
Block a user