[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+35
View File
@@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes
def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling():
"""Cudagraph memory profiling uses a minimal KV cache, so the Mamba
block-count guard must only fire for the real cache sizing."""
kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"):
compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
)
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE,
)
cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.ALWAYS,
"FakeAttentionBackend",
uniform_decode_query_len=1,
use_v2_model_runner=True,
tensor_parallel_size=1,
kv_cache_config=kv_cache_config,
max_num_reqs=256,
is_profiling=True,
)
assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE
@pytest.mark.parametrize( @pytest.mark.parametrize(
("model_config", "expected"), ("model_config", "expected"),
[ [
+4 -3
View File
@@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
"facebook/opt-125m", "facebook/opt-125m",
max_logprobs=5, max_logprobs=5,
enable_prefix_caching=False, enable_prefix_caching=False,
# 2 other llms alive during whole session # 2 other llms alive during whole session; must also cover the
gpu_memory_utilization=0.05, # cudagraph memory reservation from startup profiling.
gpu_memory_utilization=0.1,
max_model_len=16, max_model_len=16,
logprobs_mode=logprobs_mode, logprobs_mode=logprobs_mode,
) )
@@ -575,7 +576,7 @@ def test_prompt_logprobs_mode():
llm = LLM( llm = LLM(
"facebook/opt-125m", "facebook/opt-125m",
enable_prefix_caching=False, enable_prefix_caching=False,
gpu_memory_utilization=0.05, gpu_memory_utilization=0.1,
max_model_len=16, max_model_len=16,
logprobs_mode=mode, logprobs_mode=mode,
) )
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for GPUModelRunner (V2) CUDA graph memory profiling.
These exercise the orchestration of ``profile_cudagraph_memory`` on CPU by
building a runner via ``__new__`` and faking the GPU-only helpers, so the
control flow (bootstrap -> sample FULL graphs into a throwaway pool ->
extrapolate -> teardown) is covered without a GPU.
See https://github.com/vllm-project/vllm/issues/49224.
"""
import contextlib
from types import SimpleNamespace
from typing import Any
from vllm.compilation.counter import compilation_counter
from vllm.config.compilation import CUDAGraphMode
from vllm.v1.worker.gpu import cudagraph_utils as cgu
from vllm.v1.worker.gpu import model_runner as mrv2
GLOBAL_POOL = "global-pool"
THROWAWAY_POOL = "throwaway-pool"
class _FakeCudaGraphManager:
def __init__(
self, needs_capture: bool, num_full_descs: int, piecewise_only: bool = False
) -> None:
self._needs_capture = needs_capture
self.pool: Any = GLOBAL_POOL
descs = [object() for _ in range(num_full_descs)]
if piecewise_only:
self._capture_descs = {CUDAGraphMode.PIECEWISE: descs}
else:
self._capture_descs = {CUDAGraphMode.FULL: descs} if needs_capture else {}
# Profiling hooks set by profile_cudagraph_memory.
self._max_full_descs_to_capture: int | None = None
self._capture_mem_samples: list[int] | None = None
self.use_breakable_cg = False
def needs_capture(self) -> bool:
return self._needs_capture
def _make_profiling_runner(
cudagraph_mode: CUDAGraphMode,
*,
needs_capture: bool = True,
num_full_descs: int = 3,
piecewise_only: bool = False,
captured_bytes: int = 7 << 30,
mem_samples: list[int] | None = None,
) -> Any:
runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
runner.compilation_config = SimpleNamespace(cudagraph_mode=cudagraph_mode)
runner.cudagraph_manager = _FakeCudaGraphManager(
needs_capture, num_full_descs, piecewise_only
)
runner.vllm_config = SimpleNamespace()
events: list[str] = []
runner.events = events
def _capture_model() -> int:
events.append("capture")
# Simulate the manager's per-FULL-graph memory sampling.
samples = runner.cudagraph_manager._capture_mem_samples
if samples is not None:
samples.extend(mem_samples or [])
return captured_bytes
runner.capture_model = _capture_model
return runner
def _patch_module(monkeypatch) -> None:
@contextlib.contextmanager
def _fake_set_current_vllm_config(_cfg):
yield
monkeypatch.setattr(cgu, "set_current_vllm_config", _fake_set_current_vllm_config)
monkeypatch.setattr(
cgu,
"current_platform",
SimpleNamespace(graph_pool_handle=lambda: THROWAWAY_POOL),
)
monkeypatch.setattr(
cgu, "_init_minimal_kv_cache_for_profiling", lambda r: r.events.append("init")
)
monkeypatch.setattr(
cgu, "_teardown_profiling_state", lambda r: r.events.append("teardown")
)
# The profiler reads free GPU memory before/after to compute what it
# retained; default to a constant (nothing retained).
monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None)
monkeypatch.setattr(
cgu.torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30)
)
def test_profile_cudagraph_memory_disabled_returns_zero(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.NONE)
result = cgu.profile_cudagraph_memory(runner)
assert result == 0
# No KV-cache bootstrap or teardown when cudagraphs are disabled.
assert runner.events == []
def test_profile_cudagraph_memory_no_graphs_tears_down(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL, needs_capture=False)
result = cgu.profile_cudagraph_memory(runner)
assert result == 0
# Bootstrapped then cleaned up, without capturing or touching the pool.
assert runner.events == ["init", "teardown"]
assert runner.cudagraph_manager.pool == GLOBAL_POOL
def test_profile_cudagraph_memory_samples_and_extrapolates(monkeypatch):
_patch_module(monkeypatch)
gib = 1 << 30
# Measured delta 1000 MiB includes the sampled FULL graphs (100 + 20 MiB).
# Extrapolated FULL cost for 3 graphs: 100 + 2 * 20 = 140 MiB.
runner = _make_profiling_runner(
CUDAGraphMode.FULL,
num_full_descs=3,
captured_bytes=1000 * gib,
mem_samples=[100 * gib, 20 * gib],
)
result = cgu.profile_cudagraph_memory(runner)
assert result == (1000 - (100 + 20) + (100 + 2 * 20)) * gib
# Bootstrap, capture, and teardown run in order.
assert runner.events == ["init", "capture", "teardown"]
# Capture must use a throwaway pool, not the persistent global pool.
assert runner.cudagraph_manager.pool == THROWAWAY_POOL
# FULL capture must be limited to the largest few graphs.
assert (
runner.cudagraph_manager._max_full_descs_to_capture
== cgu._FULL_GRAPH_PROFILING_SAMPLES
)
def test_profile_cudagraph_memory_piecewise_only_returns_measured(monkeypatch):
_patch_module(monkeypatch)
captured_bytes = 5 << 30
runner = _make_profiling_runner(
CUDAGraphMode.FULL_AND_PIECEWISE,
piecewise_only=True,
captured_bytes=captured_bytes,
)
result = cgu.profile_cudagraph_memory(runner)
# No FULL graphs to sample or extrapolate: the measured delta is exact.
assert result == captured_bytes
def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _boom() -> int:
runner.events.append("capture")
raise RuntimeError("capture failed")
runner.capture_model = _boom
try:
cgu.profile_cudagraph_memory(runner)
except RuntimeError:
pass
else:
raise AssertionError("expected capture error to propagate")
# Teardown still runs even if capture raises.
assert runner.events == ["init", "capture", "teardown"]
def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL)
def _capture_model() -> int:
compilation_counter.num_cudagraph_captured += 5
compilation_counter.num_gpu_runner_capture_triggers += 1
return 1 << 30
runner.capture_model = _capture_model
captured_before = compilation_counter.num_cudagraph_captured
triggers_before = compilation_counter.num_gpu_runner_capture_triggers
cgu.profile_cudagraph_memory(runner)
# Profiling captures are discarded, so they must not inflate the
# compilation counters; the real capture_model() runs later.
assert compilation_counter.num_cudagraph_captured == captured_before
assert compilation_counter.num_gpu_runner_capture_triggers == triggers_before
def test_model_runner_delegates_to_cudagraph_utils(monkeypatch):
runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner)
monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42)
assert runner.profile_cudagraph_memory() == 42
def test_extrapolate_full_graph_memory():
mib = 1 << 20
# No samples (e.g. no FULL graphs): nothing to add.
assert cgu._extrapolate_full_graph_memory([], 0) == 0
# A single graph costs exactly its sample.
assert cgu._extrapolate_full_graph_memory([100 * mib], 1) == 100 * mib
# First capture + per-graph cost for the rest.
assert (
cgu._extrapolate_full_graph_memory([100 * mib, 20 * mib], 5)
== (100 + 4 * 20) * mib
)
# Per-graph cost is floored to account for driver overhead.
assert cgu._extrapolate_full_graph_memory([100 * mib, 0], 3) == (100 + 2 * 1) * mib
def test_profile_cudagraph_memory_clears_captured_graphs(monkeypatch):
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
cleared: list[str] = []
monkeypatch.setattr(
cgu.CUDAGraphWrapper,
"clear_all_graphs",
classmethod(lambda cls: cleared.append("piecewise")),
)
monkeypatch.setattr(
cgu.BreakableCUDAGraphWrapper,
"clear_all_graphs",
classmethod(lambda cls: cleared.append("breakable")),
)
cgu.profile_cudagraph_memory(runner)
# Profiling captures are discarded so the real capture re-captures them
# against the KV cache.
assert cleared == ["piecewise", "breakable"]
def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch):
"""Piecewise wrappers must capture into the throwaway pool too.
Profiling graphs captured into the persistent global pool and then
discarded drop the pool's use_count to 0, tripping the c10 allocator's
create_or_incref_pool assert when the real capture reuses that pool.
"""
_patch_module(monkeypatch)
runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE)
class _FakeWrapper:
def __init__(self) -> None:
self.graph_pool: Any = GLOBAL_POOL
self.pool_during_capture: Any = None
def clear_graphs(self) -> None:
pass
wrapper = _FakeWrapper()
cgu.CUDAGraphWrapper._all_instances.add(wrapper)
try:
capture_model = runner.capture_model
def _capture_model() -> int:
wrapper.pool_during_capture = wrapper.graph_pool
return capture_model()
runner.capture_model = _capture_model
cgu.profile_cudagraph_memory(runner)
assert wrapper.pool_during_capture == THROWAWAY_POOL
assert wrapper.graph_pool == GLOBAL_POOL
finally:
cgu.CUDAGraphWrapper._all_instances.discard(wrapper)
@@ -416,7 +416,10 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention):
) )
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata_narrowed = attn_metadata_raw[self.prefix] attn_metadata_narrowed = attn_metadata_raw.get(self.prefix)
if attn_metadata_narrowed is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata) assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata)
m = attn_metadata_narrowed m = attn_metadata_narrowed
has_initial_state = m.has_initial_state has_initial_state = m.has_initial_state
@@ -305,7 +305,10 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention):
return return
assert isinstance(attn_metadata, dict) assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata[self.prefix] # type: ignore[assignment] attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment]
if attn_metadata is None:
# Profile/warmup dummy runs skip mamba-family metadata.
return
assert isinstance(attn_metadata, GDNAttentionMetadata) assert isinstance(attn_metadata, GDNAttentionMetadata)
has_initial_state = attn_metadata.has_initial_state has_initial_state = attn_metadata.has_initial_state
spec_query_start_loc = attn_metadata.spec_query_start_loc spec_query_start_loc = attn_metadata.spec_query_start_loc
@@ -1202,13 +1202,14 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context() forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None: attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2] v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2]
self._warmup_prefill_kernels(qkvz, v_dim) self._warmup_prefill_kernels(qkvz, v_dim)
return return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata) assert isinstance(attn_metadata, GDNAttentionMetadata)
# The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved # The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved
@@ -1259,12 +1260,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context() forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None: attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkv, 0) self._warmup_prefill_kernels(mixed_qkv, 0)
return return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata) assert isinstance(attn_metadata, GDNAttentionMetadata)
if ( if (
@@ -1773,12 +1775,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
forward_context = get_forward_context() forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata attn_metadata_raw = forward_context.attn_metadata
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
if attn_metadata_raw is None: attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0) self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
return return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata) assert isinstance(attn_metadata, GDNAttentionMetadata)
mixed_qkv, output_gate_flat = mixed_qkvz.split( mixed_qkv, output_gate_flat = mixed_qkvz.split(
[qkv_size, self.value_dim // self.tp_size], dim=-1 [qkv_size, self.value_dim // self.tp_size], dim=-1
@@ -1856,12 +1859,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
) -> None: ) -> None:
forward_context = get_forward_context() forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None: attn_metadata = None
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is None:
self._warmup_prefill_kernels(mixed_qkv, 0) self._warmup_prefill_kernels(mixed_qkv, 0)
return return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata) assert isinstance(attn_metadata, GDNAttentionMetadata)
if ( if (
self._can_use_fused_gdn_mtp_decode(attn_metadata) self._can_use_fused_gdn_mtp_decode(attn_metadata)
@@ -542,7 +542,8 @@ class BailingMoELinearAttention(LinearAttention):
attn_metadata = forward_context.attn_metadata attn_metadata = forward_context.attn_metadata
if attn_metadata is not None: if attn_metadata is not None:
assert isinstance(attn_metadata, dict) assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata[self.prefix] # type: ignore attn_metadata = attn_metadata.get(self.prefix) # type: ignore
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata) assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = ( num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
@@ -282,7 +282,8 @@ class MiniMaxText01LinearAttention(LinearAttention):
attn_metadata: AttentionMetadata | None = None attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None: if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, LinearAttentionMetadata) assert isinstance(attn_metadata, LinearAttentionMetadata)
num_actual_tokens = ( num_actual_tokens = (
attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens
@@ -269,7 +269,8 @@ class MambaMixer(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None: if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba1AttentionMetadata) assert isinstance(attn_metadata, Mamba1AttentionMetadata)
query_start_loc_p = attn_metadata.query_start_loc_p query_start_loc_p = attn_metadata.query_start_loc_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p state_indices_tensor_p = attn_metadata.state_indices_tensor_p
@@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None: if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, Mamba2AttentionMetadata) assert isinstance(attn_metadata, Mamba2AttentionMetadata)
# conv_state must be (..., dim, width-1) for the conv kernels. # conv_state must be (..., dim, width-1) for the conv kernels.
# DS layout stores it that way directly; SD layout needs a # DS layout stores it that way directly; SD layout needs a
@@ -113,8 +113,10 @@ class ShortConv(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None: if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] attn_metadata = attn_metadata_raw.get(self.prefix)
assert isinstance(attn_metadata, ShortConvAttentionMetadata) assert attn_metadata is None or isinstance(
attn_metadata, ShortConvAttentionMetadata
)
BCx, _ = self.in_proj(hidden_states) BCx, _ = self.in_proj(hidden_states)
B, C, x = BCx.chunk(3, dim=-1) B, C, x = BCx.chunk(3, dim=-1)
@@ -225,7 +227,8 @@ class ShortConv(MambaBase, PluggableLayer):
attn_metadata: AttentionMetadata | None = None attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None: if attn_metadata_raw is not None:
assert isinstance(attn_metadata_raw, dict) assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] attn_metadata = attn_metadata_raw.get(self.prefix)
if attn_metadata is not None:
assert isinstance(attn_metadata, ShortConvAttentionMetadata) assert isinstance(attn_metadata, ShortConvAttentionMetadata)
conv_state = ( conv_state = (
self.kv_cache[0] self.kv_cache[0]
+187 -2
View File
@@ -1,10 +1,11 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from collections import defaultdict from collections import defaultdict
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from itertools import product from itertools import product
from typing import Any, NamedTuple, Protocol from typing import TYPE_CHECKING, Any, NamedTuple, Protocol
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -15,7 +16,8 @@ from vllm.compilation.breakable_cudagraph import (
is_breakable_cudagraph_enabled, is_breakable_cudagraph_enabled,
) )
from vllm.compilation.counter import compilation_counter from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CUDAGraphMode from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.distributed.parallel_state import ( from vllm.distributed.parallel_state import (
@@ -37,6 +39,9 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
logger = init_logger(__name__) logger = init_logger(__name__)
@@ -132,6 +137,12 @@ class CudaGraphManager:
self._graphs_captured = False self._graphs_captured = False
# Profiling hooks, set only by profile_cudagraph_memory() below: cap
# FULL-mode capture at the N largest descriptors and record each
# captured FULL graph's memory delta for extrapolation.
self._max_full_descs_to_capture: int | None = None
self._capture_mem_samples: list[int] | None = None
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {} self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {} self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
@@ -327,6 +338,14 @@ class CudaGraphManager:
continue continue
descs = self._capture_descs[mode] descs = self._capture_descs[mode]
if (
mode == CUDAGraphMode.FULL
and self._max_full_descs_to_capture is not None
):
# Profiling only: capture a sample of the largest FULL
# graphs; the total cost is extrapolated from their
# per-graph memory deltas.
descs = descs[: self._max_full_descs_to_capture]
if is_global_first_rank(): if is_global_first_rank():
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})") descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
for desc in descs: for desc in descs:
@@ -362,6 +381,9 @@ class CudaGraphManager:
set_graph_pool_id(self.pool) set_graph_pool_id(self.pool)
else: else:
set_graph_pool_id(current_platform.graph_pool_handle()) set_graph_pool_id(current_platform.graph_pool_handle())
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_before = torch.accelerator.get_memory_info()[0]
with torch.cuda.graph(graph, self.pool): with torch.cuda.graph(graph, self.pool):
forward_fn(CUDAGraphMode.NONE) forward_fn(CUDAGraphMode.NONE)
# Join offloader's copy stream after forward to avoid # Join offloader's copy stream after forward to avoid
@@ -369,6 +391,10 @@ class CudaGraphManager:
# forks copy_stream, but wait_prefetch only happens in # forks copy_stream, but wait_prefetch only happens in
# the next forward pass. # the next forward pass.
get_offloader().join_after_forward() get_offloader().join_after_forward()
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_after = torch.accelerator.get_memory_info()[0]
self._capture_mem_samples.append(free_before - free_after)
self.graphs[desc] = graph self.graphs[desc] = graph
compilation_counter.num_cudagraph_captured += 1 compilation_counter.num_cudagraph_captured += 1
self._graphs_captured = True self._graphs_captured = True
@@ -668,3 +694,162 @@ def prepare_inputs_to_capture(
for_capture=full_cudagraph, for_capture=full_cudagraph,
) )
return AttentionState(attn_metadata, slot_mappings_by_layer) return AttentionState(attn_metadata, slot_mappings_by_layer)
# ---------------------------------------------------------------------------
# CUDA graph memory profiling
# ---------------------------------------------------------------------------
# Number of FULL graphs captured during profiling; the total FULL capture
# cost is extrapolated from this sample to avoid a second full capture.
_FULL_GRAPH_PROFILING_SAMPLES = 2
# Floor for the extrapolated per-graph cost (driver overhead per graph).
_MIN_PER_GRAPH_BYTES = 1 << 20
@torch.inference_mode()
def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
"""Estimate the GPU memory needed for CUDA graph capture.
Called during memory profiling, *before* the real KV cache is allocated,
so that ``Worker.determine_available_memory`` can reserve headroom for
graph capture. Bootstraps a minimal KV cache, runs ``capture_model()``
once, then releases everything so the real init/capture path starts clean.
FULL graphs bake in KV cache pointers, so only the largest few are
captured (into a throwaway pool) and their total cost is extrapolated.
PIECEWISE, encoder and speculator graphs are measured in full. All
profiling captures are discarded afterwards: replaying graphs recorded
against the throwaway profiling state is unsafe (e.g. inductor graph
partition reclaims the storages of earlier cudagraph recordings once the
real capture records new ones, leading to use-after-free crashes).
"""
if runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
return 0
gc.collect()
torch.accelerator.empty_cache()
with set_current_vllm_config(runner.vllm_config):
_init_minimal_kv_cache_for_profiling(runner)
manager = runner.cudagraph_manager
assert manager is not None
# Don't count profiling captures; the real capture_model() runs later.
saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured
saved_capture_triggers = compilation_counter.num_gpu_runner_capture_triggers
all_wrappers: list[Any] = []
original_pools: dict[int, Any] = {}
try:
if not manager.needs_capture():
return 0
# Capture all profiling graphs into a throwaway pool so their memory
# is reclaimed on teardown rather than retained by the persistent
# global pool (which the real capture reuses). This must include the
# piecewise wrappers, not just the FULL-graph manager pool: graphs
# captured into the global pool and then discarded drop its use_count
# to 0, and the real capture on the same pool trips the c10 allocator's
# create_or_incref_pool assert ("use_count > 0 INTERNAL ASSERT FAILED").
manager.pool = current_platform.graph_pool_handle()
if manager.use_breakable_cg:
# The breakable runner is otherwise created lazily during capture,
# after the pool swap below, and would capture into the global
# pool. Create it now so its pool gets swapped too.
manager.init_breakable_cg_runner(runner.model)
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
)
for wrapper in all_wrappers:
original_pools[id(wrapper)] = wrapper.graph_pool
wrapper.graph_pool = manager.pool
manager._max_full_descs_to_capture = _FULL_GRAPH_PROFILING_SAMPLES
mem_samples: list[int] = []
manager._capture_mem_samples = mem_samples
measured = int(runner.capture_model())
# The measured delta covers PIECEWISE, encoder and speculator graphs
# plus the sampled FULL graphs; swap the sampled FULL cost for the
# extrapolated total. FULL and PIECEWISE share one pool here just as
# they share the global pool at runtime, so the overlap is not
# double-counted.
num_full_graphs = len(manager._capture_descs.get(CUDAGraphMode.FULL, []))
full_estimate = _extrapolate_full_graph_memory(mem_samples, num_full_graphs)
return max(measured - sum(mem_samples) + full_estimate, 0)
finally:
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
compilation_counter.num_gpu_runner_capture_triggers = saved_capture_triggers
CUDAGraphWrapper.clear_all_graphs()
BreakableCUDAGraphWrapper.clear_all_graphs()
for wrapper in all_wrappers:
if id(wrapper) in original_pools:
wrapper.graph_pool = original_pools[id(wrapper)]
_teardown_profiling_state(runner)
def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int:
"""Extrapolate the total FULL capture cost from samples of the largest
graphs. The first capture allocates the pool baseline; later graphs mostly
reuse it, so the second sample is taken as the per-graph cost."""
if not mem_samples:
return 0
first_capture = mem_samples[0]
per_graph = max(mem_samples[1], _MIN_PER_GRAPH_BYTES) if len(mem_samples) > 1 else 0
return first_capture + (total_graphs - 1) * per_graph
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
"""Allocate the smallest KV cache that still lets every graph be captured."""
from vllm.v1.core.kv_cache_utils import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
kv_cache_spec = runner.get_kv_cache_spec()
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
# At least one block per sequence is required to capture the graphs.
min_blocks = (
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
or 1
)
saved_override = runner.cache_config.num_gpu_blocks_override
runner.cache_config.num_gpu_blocks_override = min_blocks
try:
minimal_config = get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, available_memory=0
)
finally:
runner.cache_config.num_gpu_blocks_override = saved_override
runner.initialize_kv_cache(minimal_config, is_profiling=True)
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
"""Release the profiling KV cache and captured graphs while keeping model
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
torch.accelerator.synchronize()
if hasattr(runner, "kv_caches"):
runner.kv_caches.clear()
if hasattr(runner, "attn_groups"):
runner.attn_groups.clear()
if hasattr(runner, "kv_cache_config"):
del runner.kv_cache_config
# Dropping the manager releases the profiling graphs and throwaway pool.
runner.cudagraph_manager = None
# Release encoder graphs captured during profiling; the real
# capture_model() re-captures them.
if runner.model_state.supports_mm_inputs:
runner.model_state.encoder_runner.clear()
# Detach profiling KV tensors held by attention layers.
for layer in runner.compilation_config.static_forward_context.values():
if hasattr(layer, "kv_cache"):
kv_cache = layer.kv_cache
layer.kv_cache = (
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
)
runner.cache_config.num_gpu_blocks = None
runner.maybe_remove_all_loras(runner.lora_config)
gc.collect()
torch.accelerator.empty_cache()
+23 -5
View File
@@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor, BatchExecutionDescriptor,
ModelCudaGraphManager, ModelCudaGraphManager,
) )
from vllm.v1.worker.gpu.cudagraph_utils import (
profile_cudagraph_memory as _profile_cudagraph_memory,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.ec_connector import get_ec_connector from vllm.v1.worker.gpu.ec_connector import get_ec_connector
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
@@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_kv_cache_spec(self): def get_kv_cache_spec(self):
return get_kv_cache_spec(self.vllm_config) return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: def initialize_kv_cache(
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
) -> None:
kv_cache_config = deepcopy(kv_cache_config) kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config self.kv_cache_config = kv_cache_config
@@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
tensor_parallel_size=self.parallel_config.tensor_parallel_size, tensor_parallel_size=self.parallel_config.tensor_parallel_size,
kv_cache_config=self.kv_cache_config, kv_cache_config=self.kv_cache_config,
max_num_reqs=self.max_num_reqs, max_num_reqs=self.max_num_reqs,
is_profiling=is_profiling,
) )
self.cudagraph_manager = ModelCudaGraphManager( self.cudagraph_manager = ModelCudaGraphManager(
self.vllm_config, self.vllm_config,
@@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.kernel_block_sizes, self.kernel_block_sizes,
self.vllm_config, self.vllm_config,
) )
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict) if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
else:
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
def _init_kv_zero_meta(self) -> None: def _init_kv_zero_meta(self) -> None:
"""Build KV-block zeroing metadata; invoked from gpu_worker.""" """Build KV-block zeroing metadata; invoked from gpu_worker."""
@@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None: if self.pooling_runner is not None:
self.pooling_runner.clear() self.pooling_runner.clear()
@torch.inference_mode()
def profile_cudagraph_memory(self) -> int: def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not. """Estimate the GPU memory required to capture CUDA graphs."""
return 0 return _profile_cudagraph_memory(self)
@torch.inference_mode() @torch.inference_mode()
def capture_model(self) -> int: def capture_model(self) -> int:
@@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings, self.kv_cache_config slot_mappings, self.kv_cache_config
) )
assert block_tables is not None assert block_tables is not None
attn_groups = self.attn_groups
if dummy_run and is_profile:
# Mamba layers take a cheap warmup path with no metadata;
# attention metadata is still built so those kernels tune.
attn_groups = [
[g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)]
for groups in attn_groups
]
attn_metadata = self.model_state.prepare_attn( attn_metadata = self.model_state.prepare_attn(
input_batch, input_batch,
batch_desc.cg_mode, batch_desc.cg_mode,
block_tables, block_tables,
slot_mappings, slot_mappings,
self.attn_groups, attn_groups,
self.kv_cache_config, self.kv_cache_config,
# FULL replay reads capture-time metadata buffers. Re-stage them # FULL replay reads capture-time metadata buffers. Re-stage them
# from the zeroed dummy block tables instead of retaining state # from the zeroed dummy block tables instead of retaining state