[Profiler] Add per-session profiling controls to Python and Rust frontends (#57875)
Signed-off-by: czhu <conway.zhu@cohere.com> Signed-off-by: Conway Zhu <conway.zhu@cohere.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, Mock, call, patch
|
||||
@@ -27,6 +28,7 @@ from vllm.profiler.wrapper import (
|
||||
validate_worker_profiler_config,
|
||||
)
|
||||
from vllm.v1.core.sched.output import CachedRequestData
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
from vllm.v1.worker.xpu_worker import XPUWorker
|
||||
@@ -159,13 +161,33 @@ def test_torch_profiler_device_summary(tmp_path, capsys, activities, dump_device
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
summary = tmp_path / "profiler_out_0.txt"
|
||||
summary = tmp_path / "worker.profiler_out.txt"
|
||||
assert summary.exists() == dump_device_time
|
||||
assert ("device times" in capsys.readouterr().out) == dump_device_time
|
||||
if dump_device_time:
|
||||
assert summary.read_text() == "device times\n"
|
||||
|
||||
|
||||
def test_torch_profiler_summary_follows_output_name(tmp_path):
|
||||
config = ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
)
|
||||
with patch("vllm.profiler.wrapper.torch.profiler.profile") as profile:
|
||||
table = profile.return_value.key_averages.return_value.table
|
||||
table.side_effect = ["first run", "second run"]
|
||||
wrapper = TorchProfilerWrapper(
|
||||
config, worker_name="initial", local_rank=1, activities=["CUDA"]
|
||||
)
|
||||
for output_name in ["first_rank1", "second_rank1"]:
|
||||
wrapper.set_output_name(output_name)
|
||||
wrapper.start()
|
||||
wrapper.stop()
|
||||
|
||||
assert (tmp_path / "first_rank1.profiler_out.txt").read_text() == "first run\n"
|
||||
assert (tmp_path / "second_rank1.profiler_out.txt").read_text() == "second run\n"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activities",
|
||||
[[], ["CPU", "CPU"], ["INVALID"]],
|
||||
@@ -184,6 +206,30 @@ def test_torch_profiler_activities_require_torch_profiler():
|
||||
ProfilerConfig(torch_profiler_activities=["CPU"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("profiler", "activities", "ignore_frontend", "expected"),
|
||||
[
|
||||
("torch", None, False, True),
|
||||
("torch", ["CPU"], False, True),
|
||||
("torch", ["CPU", "CUDA"], False, True),
|
||||
("torch", ["CUDA"], False, False),
|
||||
("torch", ["CPU"], True, False),
|
||||
("cuda", None, False, False),
|
||||
],
|
||||
)
|
||||
def test_should_profile_frontend(
|
||||
profiler, activities, ignore_frontend, expected, tmp_path
|
||||
):
|
||||
config = ProfilerConfig(
|
||||
profiler=profiler,
|
||||
torch_profiler_dir=str(tmp_path) if profiler == "torch" else "",
|
||||
torch_profiler_activities=activities,
|
||||
ignore_frontend=ignore_frontend,
|
||||
)
|
||||
|
||||
assert config.should_profile_frontend is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device_type", "activities"),
|
||||
[
|
||||
@@ -270,16 +316,81 @@ def test_worker_reuses_torch_wrapper_across_profile_rounds(worker_type):
|
||||
patch("vllm.distributed.utils.get_worker_rank_suffix", return_value="rank0"),
|
||||
patch("vllm.profiler.wrapper.TorchProfilerWrapper") as wrapper,
|
||||
):
|
||||
worker.profile()
|
||||
worker.profile(
|
||||
profile_prefix="first",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
worker.profile(is_start=False)
|
||||
worker.profile()
|
||||
worker.profile(profile_prefix="second")
|
||||
|
||||
assert worker.profiler is wrapper.return_value
|
||||
wrapper.assert_called_once()
|
||||
assert wrapper.return_value.start.call_count == 2
|
||||
assert wrapper.return_value.set_output_name.call_args_list == [
|
||||
call("first_rank0"),
|
||||
call("second_rank0"),
|
||||
]
|
||||
assert wrapper.return_value.start.call_args_list == [
|
||||
call(delay_iterations=5, max_iterations=2),
|
||||
call(delay_iterations=None, max_iterations=None),
|
||||
]
|
||||
wrapper.return_value.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_executor_profile_only_forwards_supplied_override_kwargs():
|
||||
executor = MagicMock()
|
||||
executor.vllm_config.parallel_config.worker_cls = "vllm.v1.worker.gpu_worker.Worker"
|
||||
|
||||
Executor.profile(executor, True, "default")
|
||||
Executor.profile(
|
||||
executor,
|
||||
True,
|
||||
"bounded",
|
||||
delay_iterations=5,
|
||||
max_iterations=2,
|
||||
)
|
||||
|
||||
assert executor.collective_rpc.call_args_list == [
|
||||
call("profile", args=(True, "default"), kwargs=None),
|
||||
call(
|
||||
"profile",
|
||||
args=(True, "bounded"),
|
||||
kwargs={"delay_iterations": 5, "max_iterations": 2},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class _LegacyProfileWorker:
|
||||
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
|
||||
pass
|
||||
|
||||
|
||||
def test_executor_rejects_overrides_unsupported_by_worker():
|
||||
"""Out-of-tree workers with the old ``profile`` signature are rejected
|
||||
before dispatch, so a failed collective_rpc cannot leave replies queued."""
|
||||
executor = MagicMock()
|
||||
executor.vllm_config.parallel_config.worker_cls = f"{__name__}._LegacyProfileWorker"
|
||||
|
||||
with pytest.raises(ValueError, match="max_iterations"):
|
||||
Executor.profile(executor, True, None, max_iterations=2)
|
||||
Executor.profile(executor, True, "legacy")
|
||||
|
||||
executor.collective_rpc.assert_called_once_with(
|
||||
"profile", args=(True, "legacy"), kwargs=None
|
||||
)
|
||||
|
||||
|
||||
def test_worker_rejects_invalid_profile_prefix():
|
||||
worker = object.__new__(Worker)
|
||||
worker.profiler = None
|
||||
worker.profiler_config = ProfilerConfig(
|
||||
profiler="torch", torch_profiler_dir="/tmp/mock"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="profile_prefix"):
|
||||
worker.profile(profile_prefix="../trace")
|
||||
|
||||
|
||||
def test_immediate_start_stop(default_profiler_config):
|
||||
"""Test standard start without delay."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
@@ -294,6 +405,33 @@ def test_immediate_start_stop(default_profiler_config):
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_stop_waits_for_inflight_start(default_profiler_config):
|
||||
"""A stop() from another thread during a slow start() must still stop the
|
||||
backend instead of leaving it running with the session marked inactive."""
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
in_start = threading.Event()
|
||||
release_start = threading.Event()
|
||||
|
||||
def slow_start():
|
||||
in_start.set()
|
||||
release_start.wait()
|
||||
|
||||
with patch.object(profiler, "_start", side_effect=slow_start):
|
||||
start_thread = threading.Thread(target=profiler.start)
|
||||
start_thread.start()
|
||||
assert in_start.wait(timeout=5)
|
||||
stop_thread = threading.Thread(target=profiler.stop)
|
||||
stop_thread.start()
|
||||
stop_thread.join(timeout=0.1)
|
||||
release_start.set()
|
||||
start_thread.join(timeout=5)
|
||||
stop_thread.join(timeout=5)
|
||||
|
||||
assert profiler._active is False
|
||||
assert profiler._running is False
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_delayed_start(default_profiler_config):
|
||||
"""Test that profiler waits for N steps before actually starting."""
|
||||
default_profiler_config.delay_iterations = 2
|
||||
@@ -400,19 +538,37 @@ def test_delayed_start_and_max_iters(default_profiler_config):
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_idempotency(default_profiler_config):
|
||||
"""Test that calling start/stop multiple times doesn't break logic."""
|
||||
def test_duplicate_start_is_ignored(default_profiler_config):
|
||||
profiler = ConcreteWorkerProfiler(default_profiler_config)
|
||||
|
||||
# Double Start
|
||||
profiler.start()
|
||||
profiler.start()
|
||||
assert profiler.start_call_count == 1 # Should only start once
|
||||
assert profiler.start_call_count == 1
|
||||
|
||||
# Double Stop
|
||||
profiler.stop()
|
||||
profiler.stop()
|
||||
assert profiler.stop_call_count == 1 # Should only stop once
|
||||
assert profiler.stop_call_count == 1
|
||||
|
||||
|
||||
def test_session_bounds_override_and_reset_to_defaults():
|
||||
profiler = ConcreteWorkerProfiler(
|
||||
ProfilerConfig(
|
||||
profiler="torch",
|
||||
torch_profiler_dir="/tmp/mock",
|
||||
delay_iterations=3,
|
||||
max_iterations=4,
|
||||
)
|
||||
)
|
||||
|
||||
profiler.start(delay_iterations=1, max_iterations=2)
|
||||
assert profiler._delay_iters == 1
|
||||
assert profiler._max_iters == 2
|
||||
profiler.stop()
|
||||
|
||||
profiler.start()
|
||||
assert profiler._delay_iters == 3
|
||||
assert profiler._max_iters == 4
|
||||
profiler.stop()
|
||||
|
||||
|
||||
def test_step_inactive(default_profiler_config):
|
||||
@@ -1095,7 +1251,9 @@ def test_gpu_worker_creates_proton_profiler():
|
||||
Worker.profile(worker)
|
||||
|
||||
wrapper.assert_called_once_with(worker.profiler_config, worker_name="rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.start.assert_called_once_with(
|
||||
delay_iterations=None, max_iterations=None
|
||||
)
|
||||
|
||||
|
||||
@_requires_cuda_for_proton
|
||||
@@ -1141,7 +1299,9 @@ def test_gpu_worker_reuses_cuda_graph_proton_session():
|
||||
Worker.profile(worker, is_start=False)
|
||||
|
||||
worker.profiler.set_output_name.assert_called_once_with("first_rank1")
|
||||
worker.profiler.start.assert_called_once_with()
|
||||
worker.profiler.start.assert_called_once_with(
|
||||
delay_iterations=None, max_iterations=None
|
||||
)
|
||||
worker.profiler.stop.assert_called_once_with()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user