[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+35
View File
@@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
block-count guard must only fire for the real cache sizing."""
kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"):
compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
is_profiling=True,
)
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
@pytest.mark.parametrize(
("model_config", "expected"),
[
+4 -3
View File
@@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
"facebook/opt-125m",
max_logprobs=5,
enable_prefix_caching=False,
# 2 other llms alive during whole session
gpu_memory_utilization=0.05,
# 2 other llms alive during whole session; must also cover the
# cudagraph memory reservation from startup profiling.
gpu_memory_utilization=0.1,
max_model_len=16,
logprobs_mode=logprobs_mode,
)
@@ -575,7 +576,7 @@ def test_prompt_logprobs_mode():
llm = LLM(
"facebook/opt-125m",
enable_prefix_caching=False,
gpu_memory_utilization=0.05,
gpu_memory_utilization=0.1,
max_model_len=16,
logprobs_mode=mode,
)
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for GPUModelRunner (V2) CUDA graph memory profiling.
These exercise the orchestration of ``profile_cudagraph_memory`` on CPU by
building a runner via ``__new__`` and faking the GPU-only helpers, so the
control flow (bootstrap -> sample FULL graphs into a throwaway pool ->
extrapolate -> teardown) is covered without a GPU.
See https://github.com/vllm-project/vllm/issues/49224.
"""
import contextlib
from types import SimpleNamespace
from typing import Any
from vllm.compilation.counter import compilation_counter
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu import cudagraph_utils as cgu
from vllm.v1.worker.gpu import model_runner as mrv2
GLOBAL_POOL = "global-pool"
THROWAWAY_POOL = "throwaway-pool"
class _FakeCudaGraphManager:
def __init__(
self, needs_capture: bool, num_full_descs: int, piecewise_only: bool = False
) -> None:
self._needs_capture = needs_capture
self.pool: Any = GLOBAL_POOL
descs = [object() for _ in range(num_full_descs)]
if piecewise_only:
self._capture_descs = {CUDAGraphMode.PIECEWISE: descs}
else:
self._capture_descs = {CUDAGraphMode.FULL: descs} if needs_capture else {}
# Profiling hooks set by profile_cudagraph_memory.
self._max_full_descs_to_capture: int | None = None
self._capture_mem_samples: list[int] | None = None
self.use_breakable_cg = False
def needs_capture(self) -> bool:
return self._needs_capture
def _make_profiling_runner(
cudagraph_mode: CUDAGraphMode,
*,
needs_capture: bool = True,
num_full_descs: int = 3,
piecewise_only: bool = False,
captured_bytes: int = 7 << 30,
mem_samples: list[int] | None = None,
) -> Any:
runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
runner.compilation_config = SimpleNamespace(cudagraph_mode=cudagraph_mode)
runner.cudagraph_manager = _FakeCudaGraphManager(
needs_capture, num_full_descs, piecewise_only
)
runner.vllm_config = SimpleNamespace()
events: list[str] = []
runner.events = events
def _capture_model() -> int:
events.append("capture")
# Simulate the manager's per-FULL-graph memory sampling.
samples = runner.cudagraph_manager._capture_mem_samples
if samples is not None:
samples.extend(mem_samples or [])
return captured_bytes
runner.capture_model = _capture_model
return runner
def _patch_module(monkeypatch) -> None:
@contextlib.contextmanager
def _fake_set_current_vllm_config(_cfg):
yield
monkeypatch.setattr(cgu, "set_current_vllm_config", _fake_set_current_vllm_config)
monkeypatch.setattr(
cgu,
"current_platform",
SimpleNamespace(graph_pool_handle=lambda: THROWAWAY_POOL),
)
monkeypatch.setattr(
cgu, "_init_minimal_kv_cache_for_profiling", lambda r: r.events.append("init")
)
monkeypatch.setattr(
cgu, "_teardown_profiling_state", lambda r: r.events.append("teardown")
)
# The profiler reads free GPU memory before/after to compute what it
# retained; default to a constant (nothing retained).
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
monkeypatch.setattr(
cgu.torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
)
def test_profile_cudagraph_memory_disabled_returns_zero(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.NONE)
result = cgu.profile_cudagraph_memory(runner)
assert result == 0
# No KV-cache bootstrap or teardown when cudagraphs are disabled.
assert runner.events == []
def test_profile_cudagraph_memory_no_graphs_tears_down(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL, needs_capture=False)
result = cgu.profile_cudagraph_memory(runner)
assert result == 0
# Bootstrapped then cleaned up, without capturing or touching the pool.
assert runner.events == ["init", "teardown"]
assert runner.cudagraph_manager.pool == GLOBAL_POOL
def test_profile_cudagraph_memory_samples_and_extrapolates(monkeypatch):
_patch_module(monkeypatch)
gib = 1 << 30
# Measured delta 1000 MiB includes the sampled FULL graphs (100 + 20 MiB).
# Extrapolated FULL cost for 3 graphs: 100 + 2 * 20 = 140 MiB.
runner = _make_profiling_runner(
CUDAGraphMode.FULL,
num_full_descs=3,
captured_bytes=1000 * gib,
mem_samples=[100 * gib, 20 * gib],
)
result = cgu.profile_cudagraph_memory(runner)
assert result == (1000 - (100 + 20) + (100 + 2 * 20)) * gib
# Bootstrap, capture, and teardown run in order.
assert runner.events == ["init", "capture", "teardown"]
# Capture must use a throwaway pool, not the persistent global pool.
assert runner.cudagraph_manager.pool == THROWAWAY_POOL
# FULL capture must be limited to the largest few graphs.
assert (
runner.cudagraph_manager._max_full_descs_to_capture
== cgu._FULL_GRAPH_PROFILING_SAMPLES
)
def test_profile_cudagraph_memory_piecewise_only_returns_measured(monkeypatch):
_patch_module(monkeypatch)
captured_bytes = 5 << 30
runner = _make_profiling_runner(
CUDAGraphMode.FULL_AND_PIECEWISE,
piecewise_only=True,
captured_bytes=captured_bytes,
)
result = cgu.profile_cudagraph_memory(runner)
# No FULL graphs to sample or extrapolate: the measured delta is exact.
assert result == captured_bytes
def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _boom() -> int:
runner.events.append("capture")
raise RuntimeError("capture failed")
runner.capture_model = _boom
try:
cgu.profile_cudagraph_memory(runner)
except RuntimeError:
pass
else:
raise AssertionError("expected capture error to propagate")
# Teardown still runs even if capture raises.
assert runner.events == ["init", "capture", "teardown"]
def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _capture_model() -> int:
compilation_counter.num_cudagraph_captured += 5
compilation_counter.num_gpu_runner_capture_triggers += 1
return 1 << 30
runner.capture_model = _capture_model
captured_before = compilation_counter.num_cudagraph_captured
triggers_before = compilation_counter.num_gpu_runner_capture_triggers
cgu.profile_cudagraph_memory(runner)
# Profiling captures are discarded, so they must not inflate the
# compilation counters; the real capture_model() runs later.
assert compilation_counter.num_cudagraph_captured == captured_before
assert compilation_counter.num_gpu_runner_capture_triggers == triggers_before
def test_model_runner_delegates_to_cudagraph_utils(monkeypatch):
runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42)
assert runner.profile_cudagraph_memory() == 42
def test_extrapolate_full_graph_memory():
mib = 1 << 20
# No samples (e.g. no FULL graphs): nothing to add.
assert cgu._extrapolate_full_graph_memory([], 0) == 0
# A single graph costs exactly its sample.
assert cgu._extrapolate_full_graph_memory([100 * mib], 1) == 100 * mib
# First capture + per-graph cost for the rest.
assert (
cgu._extrapolate_full_graph_memory([100 * mib, 20 * mib], 5)
== (100 + 4 * 20) * mib
)
# Per-graph cost is floored to account for driver overhead.
assert cgu._extrapolate_full_graph_memory([100 * mib, 0], 3) == (100 + 2 * 1) * mib
def test_profile_cudagraph_memory_clears_captured_graphs(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
cleared: list[str] = []
monkeypatch.setattr(
cgu.CUDAGraphWrapper,
"clear_all_graphs",
classmethod(lambda cls: cleared.append("piecewise")),
)
monkeypatch.setattr(
cgu.BreakableCUDAGraphWrapper,
"clear_all_graphs",
classmethod(lambda cls: cleared.append("breakable")),
)
cgu.profile_cudagraph_memory(runner)
# Profiling captures are discarded so the real capture re-captures them
# against the KV cache.
assert cleared == ["piecewise", "breakable"]
def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch):
"""Piecewise wrappers must capture into the throwaway pool too.
Profiling graphs captured into the persistent global pool and then
discarded drop the pool's use_count to 0, tripping the c10 allocator's
create_or_incref_pool assert when the real capture reuses that pool.
"""
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
class _FakeWrapper:
def __init__(self) -> None:
self.graph_pool: Any = GLOBAL_POOL
self.pool_during_capture: Any = None
def clear_graphs(self) -> None:
pass
wrapper = _FakeWrapper()
cgu.CUDAGraphWrapper._all_instances.add(wrapper)
try:
capture_model = runner.capture_model
def _capture_model() -> int:
wrapper.pool_during_capture = wrapper.graph_pool
return capture_model()
runner.capture_model = _capture_model
cgu.profile_cudagraph_memory(runner)
assert wrapper.pool_during_capture == THROWAWAY_POOL
assert wrapper.graph_pool == GLOBAL_POOL
finally:
cgu.CUDAGraphWrapper._all_instances.discard(wrapper)