[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)
Signed-off-by: czhu <conway.zhu@cohere.com> Signed-off-by: Conway Zhu <conway.zhu@cohere.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -98,6 +98,17 @@ class _FakeSession:
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
class _FakeProfileSession:
|
||||
def __init__(self) -> None:
|
||||
self.url: str | None = None
|
||||
self.headers: dict[str, str] | None = None
|
||||
|
||||
def post(self, *, url: str, headers: dict[str, str]):
|
||||
self.url = url
|
||||
self.headers = headers
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
@@ -222,6 +233,42 @@ def test_async_request_openai_audio_handles_decoded_audio_arrays(
|
||||
assert output.generated_text == "hello"
|
||||
|
||||
|
||||
def test_audio_request_rejects_profile_endpoint() -> None:
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/start_profile",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="OpenAI Audio API"):
|
||||
asyncio.run(
|
||||
request_func_module.async_request_openai_audio(
|
||||
request_input, _FakeProfileSession()
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["start_profile", "stop_profile"])
|
||||
def test_audio_benchmark_profile_request_has_no_inference_payload(
|
||||
endpoint: str,
|
||||
) -> None:
|
||||
session = _FakeProfileSession()
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_profile(
|
||||
f"http://localhost:8000/{endpoint}",
|
||||
session,
|
||||
extra_headers={"X-Test-Header": "profile"},
|
||||
)
|
||||
)
|
||||
|
||||
assert output.success is True
|
||||
assert session.url == f"http://localhost:8000/{endpoint}"
|
||||
assert session.headers == {"X-Test-Header": "profile"}
|
||||
|
||||
|
||||
_COHERE_ASR_PROMPT = (
|
||||
"<|startofcontext|><|startoftranscript|>"
|
||||
"<|emo:undefined|><|en|><|en|><|pnc|><|noitn|>"
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.serve.profile.api_router import router
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine_client() -> AsyncMock:
|
||||
return AsyncMock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(engine_client: AsyncMock) -> TestClient:
|
||||
app = FastAPI()
|
||||
app.state.engine_client = engine_client
|
||||
app.include_router(router)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_start_profile_without_body_is_backwards_compatible(
|
||||
client: TestClient, engine_client: AsyncMock
|
||||
) -> None:
|
||||
response = client.post("/start_profile")
|
||||
|
||||
assert response.status_code == 200
|
||||
engine_client.start_profile.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_start_profile_ignores_completion_payload(
|
||||
client: TestClient, engine_client: AsyncMock
|
||||
) -> None:
|
||||
response = client.post(
|
||||
"/start_profile",
|
||||
json={
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"prompt": "profile this request",
|
||||
"max_tokens": 16,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
engine_client.start_profile.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_start_profile_forwards_session_overrides(
|
||||
client: TestClient, engine_client: AsyncMock
|
||||
) -> None:
|
||||
response = client.post(
|
||||
"/start_profile",
|
||||
json={
|
||||
"profile_prefix": "sharegpt_run-1",
|
||||
"delay_iterations": 5000,
|
||||
"max_iterations": 20,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
engine_client.start_profile.assert_awaited_once_with(
|
||||
profile_prefix="sharegpt_run-1",
|
||||
delay_iterations=5000,
|
||||
max_iterations=20,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"body",
|
||||
[
|
||||
{"profile_prefix": 1},
|
||||
{"delay_iterations": "1"},
|
||||
{"max_iterations": "20"},
|
||||
],
|
||||
)
|
||||
def test_start_profile_rejects_wrongly_typed_overrides(
|
||||
client: TestClient, engine_client: AsyncMock, body: dict
|
||||
) -> None:
|
||||
response = client.post("/start_profile", json=body)
|
||||
|
||||
assert response.status_code == 422
|
||||
engine_client.start_profile.assert_not_awaited()
|
||||
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from contextlib import ExitStack
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import AsyncMock, MagicMock, call
|
||||
|
||||
import pytest
|
||||
@@ -11,7 +13,7 @@ import pytest
|
||||
import vllm.v1.engine.async_llm as async_llm_module
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.config import LoggingConfig, VllmConfig
|
||||
from vllm.config import LoggingConfig, ProfilerConfig, VllmConfig
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
@@ -20,10 +22,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.exceptions import ProfilerAlreadyActiveError, VLLMValidationError
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import WorkerProfiler
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.utils.torch_utils import set_default_torch_num_threads
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -87,13 +90,19 @@ def _mock_async_llm_dependencies(monkeypatch: pytest.MonkeyPatch):
|
||||
return engine_core
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
def _mock_vllm_config() -> MagicMock:
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.logging_config = LoggingConfig()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
return vllm_config
|
||||
|
||||
|
||||
def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.profiler = "cuda"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
@@ -105,22 +114,22 @@ def test_cuda_profiler_requests_reach_engine_core(monkeypatch: pytest.MonkeyPatc
|
||||
asyncio.run(profile())
|
||||
|
||||
assert engine.profiler is None
|
||||
engine_core.profile_async.assert_has_awaits([call(True, None), call(False)])
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[call(True, None, None, None), call(False)]
|
||||
)
|
||||
|
||||
|
||||
def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.logging_config = LoggingConfig()
|
||||
vllm_config.observability_config.otlp_traces_endpoint = None
|
||||
vllm_config.scheduler_config.stream_interval = 1
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.profiler = "torch"
|
||||
vllm_config.profiler_config.ignore_frontend = False
|
||||
vllm_config.profiler_config.torch_profiler_activities = ["CUDA"]
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "TorchProfilerWrapper", profiler)
|
||||
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
@@ -128,6 +137,271 @@ def test_cuda_only_torch_profiler_skips_frontend_cpu_trace(
|
||||
profiler.assert_not_called()
|
||||
|
||||
|
||||
def test_profile_forwards_overrides_rejects_duplicate_and_allows_restart(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config.should_profile_frontend = False
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
async def profile():
|
||||
for kwargs in (
|
||||
{"profile_prefix": "../trace"},
|
||||
{"delay_iterations": -1},
|
||||
{"max_iterations": -1},
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
await engine.start_profile(**kwargs)
|
||||
await engine.start_profile(
|
||||
"session",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
with pytest.raises(ProfilerAlreadyActiveError) as exc_info:
|
||||
await engine.start_profile("duplicate")
|
||||
assert exc_info.value.http_status == HTTPStatus.CONFLICT
|
||||
await engine.stop_profile()
|
||||
await engine.start_profile("next-session")
|
||||
await engine.stop_profile()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[
|
||||
call(True, "session", 5, 2),
|
||||
call(False),
|
||||
call(True, "next-session", None, None),
|
||||
call(False),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_stop_profile_waits_for_inflight_start(monkeypatch: pytest.MonkeyPatch):
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
events: list[str] = []
|
||||
frontend_profiler = MagicMock()
|
||||
frontend_profiler.start.side_effect = lambda: events.append("frontend_start")
|
||||
frontend_profiler.stop.side_effect = lambda: events.append("frontend_stop")
|
||||
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
start_rpc_entered = asyncio.Event()
|
||||
release_start_rpc = asyncio.Event()
|
||||
|
||||
async def profile_async(is_start, *args):
|
||||
if is_start:
|
||||
start_rpc_entered.set()
|
||||
await release_start_rpc.wait()
|
||||
events.append("engine_start_done")
|
||||
else:
|
||||
events.append("engine_stop")
|
||||
|
||||
engine_core.profile_async.side_effect = profile_async
|
||||
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
await start_rpc_entered.wait()
|
||||
|
||||
stop_task = asyncio.create_task(engine.stop_profile())
|
||||
await asyncio.sleep(0)
|
||||
release_start_rpc.set()
|
||||
await asyncio.wait_for(asyncio.gather(start_task, stop_task), timeout=5)
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
assert events[:2] == ["engine_start_done", "frontend_start"]
|
||||
assert sorted(events[2:]) == ["engine_stop", "frontend_stop"]
|
||||
assert engine._profile_session_active is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["error", "cancel"])
|
||||
def test_interrupted_engine_profile_start_skips_frontend_profiler(
|
||||
monkeypatch: pytest.MonkeyPatch, failure: str
|
||||
):
|
||||
"""The frontend trace starts only after the engine start succeeds. A
|
||||
cancelled start may still reach the engine, so it keeps the session active
|
||||
until /stop_profile."""
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
frontend_profiler = MagicMock()
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
start_rpc_entered = asyncio.Event()
|
||||
|
||||
async def profile_async(is_start, *args):
|
||||
if not is_start:
|
||||
return
|
||||
start_rpc_entered.set()
|
||||
if failure == "error":
|
||||
raise RuntimeError("engine profiler failed to start")
|
||||
await asyncio.Event().wait()
|
||||
|
||||
engine_core.profile_async.side_effect = profile_async
|
||||
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
await start_rpc_entered.wait()
|
||||
if failure == "cancel":
|
||||
start_task.cancel()
|
||||
expected = RuntimeError if failure == "error" else asyncio.CancelledError
|
||||
with pytest.raises(expected):
|
||||
await start_task
|
||||
|
||||
assert engine._profile_session_active is (failure == "cancel")
|
||||
await engine.stop_profile()
|
||||
assert engine._profile_session_active is False
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
frontend_profiler.start.assert_not_called()
|
||||
|
||||
|
||||
class _BlockingStartProfiler(WorkerProfiler):
|
||||
def __init__(self) -> None:
|
||||
self.in_start = threading.Event()
|
||||
self.release_start = threading.Event()
|
||||
self.stop_call_count = 0
|
||||
super().__init__(
|
||||
ProfilerConfig(profiler="torch", torch_profiler_dir="/tmp/mock")
|
||||
)
|
||||
|
||||
def _start(self) -> None:
|
||||
self.in_start.set()
|
||||
if not self.release_start.wait(timeout=5):
|
||||
raise TimeoutError("start was never released")
|
||||
|
||||
def _stop(self) -> None:
|
||||
self.stop_call_count += 1
|
||||
|
||||
|
||||
def test_stop_after_cancelled_frontend_start_stops_frontend_profiler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Cancelling start_profile while the frontend profiler is starting keeps
|
||||
the session active, and the next stop_profile still stops the backend."""
|
||||
vllm_config = _mock_vllm_config()
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
frontend_profiler = _BlockingStartProfiler()
|
||||
engine = AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
profiler=frontend_profiler,
|
||||
)
|
||||
|
||||
async def profile():
|
||||
try:
|
||||
start_task = asyncio.create_task(engine.start_profile())
|
||||
assert await asyncio.to_thread(frontend_profiler.in_start.wait, 5)
|
||||
start_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await start_task
|
||||
assert engine._profile_session_active is True
|
||||
|
||||
stop_task = asyncio.create_task(engine.stop_profile())
|
||||
done, _ = await asyncio.wait({stop_task}, timeout=0.1)
|
||||
assert not done, "stop_profile did not wait for the in-flight start"
|
||||
frontend_profiler.release_start.set()
|
||||
await asyncio.wait_for(stop_task, timeout=5)
|
||||
finally:
|
||||
frontend_profiler.release_start.set()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
assert frontend_profiler.stop_call_count == 1
|
||||
assert frontend_profiler._active is False
|
||||
assert frontend_profiler._running is False
|
||||
assert engine._profile_session_active is False
|
||||
assert engine_core.profile_async.await_args_list == [
|
||||
call(True, None, None, None),
|
||||
call(False),
|
||||
]
|
||||
|
||||
|
||||
def test_multi_client_profile_uses_idempotent_engine_requests(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CPU"],
|
||||
)
|
||||
engine_core = _mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr(async_llm_module, "create_frontend_profiler", profiler)
|
||||
engines = [
|
||||
AsyncLLM(
|
||||
vllm_config,
|
||||
MagicMock(),
|
||||
log_stats=False,
|
||||
client_count=2,
|
||||
client_index=client_index,
|
||||
)
|
||||
for client_index in range(2)
|
||||
]
|
||||
|
||||
assert all(engine.profiler is None for engine in engines)
|
||||
profiler.assert_not_called()
|
||||
|
||||
async def profile():
|
||||
await engines[0].start_profile()
|
||||
await engines[1].start_profile()
|
||||
await engines[1].stop_profile()
|
||||
await engines[0].start_profile()
|
||||
|
||||
asyncio.run(profile())
|
||||
|
||||
engine_core.profile_async.assert_has_awaits(
|
||||
[
|
||||
call(True, None, None, None),
|
||||
call(True, None, None, None),
|
||||
call(False),
|
||||
call(True, None, None, None),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_frontend_profiler_ignores_worker_iteration_bounds(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path
|
||||
):
|
||||
vllm_config = _mock_vllm_config()
|
||||
vllm_config.profiler_config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_activities=["CPU", "CUDA"],
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
wait_iterations=3,
|
||||
warmup_iterations=1,
|
||||
)
|
||||
_mock_async_llm_dependencies(monkeypatch)
|
||||
profiler = MagicMock()
|
||||
monkeypatch.setattr("vllm.profiler.wrapper.TorchProfilerWrapper", profiler)
|
||||
|
||||
engine = AsyncLLM(vllm_config, MagicMock(), log_stats=False)
|
||||
|
||||
frontend_config = profiler.call_args.args[0]
|
||||
assert frontend_config.delay_iterations == 0
|
||||
assert frontend_config.max_iterations == 0
|
||||
assert frontend_config.wait_iterations == 0
|
||||
assert frontend_config.warmup_iterations == 0
|
||||
assert engine.profiler is profiler.return_value
|
||||
|
||||
|
||||
async def generate(
|
||||
engine: AsyncLLM,
|
||||
request_id: str,
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, call
|
||||
import pytest
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.exceptions import ProfilerAlreadyActiveError
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
|
||||
@@ -52,6 +53,38 @@ def test_step_invalidates_mm_processor_cache_misses():
|
||||
]
|
||||
|
||||
|
||||
def test_duplicate_profile_start_rejected_before_engine_core_dispatch():
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.engine_core = MagicMock()
|
||||
engine._profile_session_active = False
|
||||
|
||||
engine.start_profile("first")
|
||||
|
||||
with pytest.raises(ProfilerAlreadyActiveError):
|
||||
engine.start_profile("duplicate")
|
||||
|
||||
engine.engine_core.profile.assert_called_once_with(True, "first", None, None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"profile_prefix": "../trace"},
|
||||
{"delay_iterations": -1},
|
||||
{"max_iterations": -1},
|
||||
],
|
||||
)
|
||||
def test_invalid_profile_controls_rejected_before_engine_core_dispatch(kwargs):
|
||||
engine = object.__new__(LLMEngine)
|
||||
engine.engine_core = MagicMock()
|
||||
engine._profile_session_active = False
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
engine.start_profile(**kwargs)
|
||||
|
||||
engine.engine_core.profile.assert_not_called()
|
||||
|
||||
|
||||
def _vllm_model(
|
||||
apc: bool,
|
||||
vllm_runner: type[VllmRunner],
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock, call, patch
|
||||
@@ -27,6 +28,7 @@ from vllm.profiler.wrapper import (
|
||||
validate_worker_profiler_config,
|
||||
)
|
||||
from vllm.v1.core.sched.output import CachedRequestData
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
from vllm.v1.worker.xpu_worker import XPUWorker
|
||||
@@ -159,13 +161,33 @@ def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
summary = tmp_path / "profiler_out_0.txt"
|
||||
summary = tmp_path / "worker.profiler_out.txt"
|
||||
assert summary.exists() == dump_device_time
|
||||
assert ("device times" in capsys.readouterr().out) == dump_device_time
|
||||
if dump_device_time:
|
||||
assert summary.read_text() == "device times\n"
|
||||
|
||||
|
||||
def test_torch_profiler_summary_follows_output_name(tmp_path):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
)
|
||||
with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile:
|
||||
table = profile.return_value.key_averages.return_value.table
|
||||
table.side_effect = ["first run", "second run"]
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="initial", local_rank=1, activities=["CUDA"]
|
||||
)
|
||||
for output_name in ["first_rank1", "second_rank1"]:
|
||||
wrapper.set_output_name(output_name)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
assert (tmp_path / "first_rank1.profiler_out.txt").read_text() == "first run\n"
|
||||
assert (tmp_path / "second_rank1.profiler_out.txt").read_text() == "second run\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities",
|
||||
[[], ["CPU", "CPU"], ["INVALID"]],
|
||||
@@ -184,6 +206,30 @@ def test_torch_profiler_activities_require_torch_profiler():
|
||||
ProfilerConfig(torch_profiler_activities=["CPU"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("profiler", "activities", "ignore_frontend", "expected"),
|
||||
[
|
||||
("torch", None, False, True),
|
||||
("torch", ["CPU"], False, True),
|
||||
("torch", ["CPU", "CUDA"], False, True),
|
||||
("torch", ["CUDA"], False, False),
|
||||
("torch", ["CPU"], True, False),
|
||||
("cuda", None, False, False),
|
||||
],
|
||||
)
|
||||
def test_should_profile_frontend(
|
||||
profiler, activities, ignore_frontend, expected, tmp_path
|
||||
):
|
||||
config = ProfilerConfig(
|
||||
profiler=profiler,
|
||||
torch_profiler_dir=str(tmp_path) if profiler == "torch" else "",
|
||||
torch_profiler_activities=activities,
|
||||
ignore_frontend=ignore_frontend,
|
||||
)
|
||||
|
||||
assert config.should_profile_frontend is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities"),
|
||||
[
|
||||
@@ -270,16 +316,81 @@ def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type):
|
||||
patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
worker.profile()
|
||||
worker.profile(
|
||||
profile_prefix="first",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
worker.profile(is_start=False)
|
||||
worker.profile()
|
||||
worker.profile(profile_prefix="second")
|
||||
|
||||
assert worker.profiler is wrapper.return_value
|
||||
wrapper.assert_called_once()
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
assert wrapper.return_value.set_output_name.call_args_list == [
|
||||
call("first_rank0"),
|
||||
call("second_rank0"),
|
||||
]
|
||||
assert wrapper.return_value.start.call_args_list == [
|
||||
call(delay_iterations=5, max_iterations=2),
|
||||
call(delay_iterations=None, max_iterations=None),
|
||||
]
|
||||
wrapper.return_value.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_executor_profile_only_forwards_supplied_override_kwargs():
|
||||
executor = MagicMock()
|
||||
executor.vllm_config.parallel_config.worker_cls = "vllm.v1.worker.gpu_worker.Worker"
|
||||
|
||||
Executor.profile(executor, True, "default")
|
||||
Executor.profile(
|
||||
executor,
|
||||
True,
|
||||
"bounded",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
|
||||
assert executor.collective_rpc.call_args_list == [
|
||||
call("profile", args=(True, "default"), kwargs=None),
|
||||
call(
|
||||
"profile",
|
||||
args=(True, "bounded"),
|
||||
kwargs={"delay_iterations": 5, "max_iterations": 2},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _LegacyProfileWorker:
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
pass
|
||||
|
||||
|
||||
def test_executor_rejects_overrides_unsupported_by_worker():
|
||||
"""Out-of-tree workers with the old ``profile`` signature are rejected
|
||||
before dispatch, so a failed collective_rpc cannot leave replies queued."""
|
||||
executor = MagicMock()
|
||||
executor.vllm_config.parallel_config.worker_cls = f"{__name__}._LegacyProfileWorker"
|
||||
|
||||
with pytest.raises(ValueError, match="max_iterations"):
|
||||
Executor.profile(executor, True, None, max_iterations=2)
|
||||
Executor.profile(executor, True, "legacy")
|
||||
|
||||
executor.collective_rpc.assert_called_once_with(
|
||||
"profile", args=(True, "legacy"), kwargs=None
|
||||
)
|
||||
|
||||
|
||||
def test_worker_rejects_invalid_profile_prefix():
|
||||
worker = object.__new__(Worker)
|
||||
worker.profiler = None
|
||||
worker.profiler_config = ProfilerConfig(
|
||||
profiler="torch", torch_profiler_dir="/tmp/mock"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="profile_prefix"):
|
||||
worker.profile(profile_prefix="../trace")
|
||||
|
||||
|
||||
def test_immediate_start_stop(default_profiler_config):
|
||||
"""Test standard start without delay."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
@@ -294,6 +405,33 @@ def test_immediate_start_stop(default_profiler_config):
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_stop_waits_for_inflight_start(default_profiler_config):
|
||||
"""A stop() from another thread during a slow start() must still stop the
|
||||
backend instead of leaving it running with the session marked inactive."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
in_start = threading.Event()
|
||||
release_start = threading.Event()
|
||||
|
||||
def slow_start():
|
||||
in_start.set()
|
||||
release_start.wait()
|
||||
|
||||
with patch.object(profiler, "_start", side_effect=slow_start):
|
||||
start_thread = threading.Thread(target=profiler.start)
|
||||
start_thread.start()
|
||||
assert in_start.wait(timeout=5)
|
||||
stop_thread = threading.Thread(target=profiler.stop)
|
||||
stop_thread.start()
|
||||
stop_thread.join(timeout=0.1)
|
||||
release_start.set()
|
||||
start_thread.join(timeout=5)
|
||||
stop_thread.join(timeout=5)
|
||||
|
||||
assert profiler._active is False
|
||||
assert profiler._running is False
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_delayed_start(default_profiler_config):
|
||||
"""Test that profiler waits for N steps before actually starting."""
|
||||
default_profiler_config.delay_iterations = 2
|
||||
@@ -400,19 +538,37 @@ def test_delayed_start_and_max_iters(default_profiler_config):
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_idempotency(default_profiler_config):
|
||||
"""Test that calling start/stop multiple times doesn't break logic."""
|
||||
def test_duplicate_start_is_ignored(default_profiler_config):
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
|
||||
# Double Start
|
||||
profiler.start()
|
||||
profiler.start()
|
||||
assert profiler.start_call_count == 1 # Should only start once
|
||||
assert profiler.start_call_count == 1
|
||||
|
||||
# Double Stop
|
||||
profiler.stop()
|
||||
profiler.stop()
|
||||
assert profiler.stop_call_count == 1 # Should only stop once
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_session_bounds_override_and_reset_to_defaults():
|
||||
profiler = ConcreteWorkerProfiler(
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
delay_iterations=3,
|
||||
max_iterations=4,
|
||||
)
|
||||
)
|
||||
|
||||
profiler.start(delay_iterations=1, max_iterations=2)
|
||||
assert profiler._delay_iters == 1
|
||||
assert profiler._max_iters == 2
|
||||
profiler.stop()
|
||||
|
||||
profiler.start()
|
||||
assert profiler._delay_iters == 3
|
||||
assert profiler._max_iters == 4
|
||||
profiler.stop()
|
||||
|
||||
|
||||
def test_step_inactive(default_profiler_config):
|
||||
@@ -1095,7 +1251,9 @@ def test_gpu_worker_creates_proton_profiler():
|
||||
Worker.profile(worker)
|
||||
|
||||
wrapper.assert_called_once_with(worker.profiler_config, worker_name="rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.start.assert_called_once_with(
|
||||
delay_iterations=None, max_iterations=None
|
||||
)
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@@ -1141,7 +1299,9 @@ def test_gpu_worker_reuses_cuda_graph_proton_session():
|
||||
Worker.profile(worker, is_start=False)
|
||||
|
||||
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.start.assert_called_once_with(
|
||||
delay_iterations=None, max_iterations=None
|
||||
)
|
||||
worker.profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user