diff --git a/tests/test_config.py b/tests/test_config.py index 40ab2fa426..b7c3740f72 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -347,6 +347,41 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes +def test_resolve_cudagraph_mode_skips_mamba_block_check_while_profiling(): + """Cudagraph memory profiling uses a minimal KV cache, so the Mamba + block-count guard must only fire for the real cache sizing.""" + kv_cache_config = SimpleNamespace(has_mamba_layers=True, num_blocks=4) + + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + ) + with pytest.raises(ValueError, match="exceeds available Mamba cache blocks"): + compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport.ALWAYS, + "FakeAttentionBackend", + uniform_decode_query_len=1, + use_v2_model_runner=True, + tensor_parallel_size=1, + kv_cache_config=kv_cache_config, + max_num_reqs=256, + ) + + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + ) + cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport.ALWAYS, + "FakeAttentionBackend", + uniform_decode_query_len=1, + use_v2_model_runner=True, + tensor_parallel_size=1, + kv_cache_config=kv_cache_config, + max_num_reqs=256, + is_profiling=True, + ) + assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE + + @pytest.mark.parametrize( ("model_config", "expected"), [ diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index f1e88bec8a..3bf26d5170 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -535,8 +535,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode): "facebook/opt-125m", max_logprobs=5, enable_prefix_caching=False, - # 2 other llms alive during whole session - gpu_memory_utilization=0.05, + # 2 other llms alive during whole session; must also cover the + # cudagraph memory reservation from startup profiling. + gpu_memory_utilization=0.1, max_model_len=16, logprobs_mode=logprobs_mode, ) @@ -575,7 +576,7 @@ def test_prompt_logprobs_mode(): llm = LLM( "facebook/opt-125m", enable_prefix_caching=False, - gpu_memory_utilization=0.05, + gpu_memory_utilization=0.1, max_model_len=16, logprobs_mode=mode, ) diff --git a/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py new file mode 100644 index 0000000000..702f371669 --- /dev/null +++ b/tests/v1/worker/test_gpu_model_runner_v2_cudagraph_profiling.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for GPUModelRunner (V2) CUDA graph memory profiling. + +These exercise the orchestration of ``profile_cudagraph_memory`` on CPU by +building a runner via ``__new__`` and faking the GPU-only helpers, so the +control flow (bootstrap -> sample FULL graphs into a throwaway pool -> +extrapolate -> teardown) is covered without a GPU. +See https://github.com/vllm-project/vllm/issues/49224. +""" + +import contextlib +from types import SimpleNamespace +from typing import Any + +from vllm.compilation.counter import compilation_counter +from vllm.config.compilation import CUDAGraphMode +from vllm.v1.worker.gpu import cudagraph_utils as cgu +from vllm.v1.worker.gpu import model_runner as mrv2 + +GLOBAL_POOL = "global-pool" +THROWAWAY_POOL = "throwaway-pool" + + +class _FakeCudaGraphManager: + def __init__( + self, needs_capture: bool, num_full_descs: int, piecewise_only: bool = False + ) -> None: + self._needs_capture = needs_capture + self.pool: Any = GLOBAL_POOL + descs = [object() for _ in range(num_full_descs)] + if piecewise_only: + self._capture_descs = {CUDAGraphMode.PIECEWISE: descs} + else: + self._capture_descs = {CUDAGraphMode.FULL: descs} if needs_capture else {} + # Profiling hooks set by profile_cudagraph_memory. + self._max_full_descs_to_capture: int | None = None + self._capture_mem_samples: list[int] | None = None + self.use_breakable_cg = False + + def needs_capture(self) -> bool: + return self._needs_capture + + +def _make_profiling_runner( + cudagraph_mode: CUDAGraphMode, + *, + needs_capture: bool = True, + num_full_descs: int = 3, + piecewise_only: bool = False, + captured_bytes: int = 7 << 30, + mem_samples: list[int] | None = None, +) -> Any: + runner: Any = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner) + runner.compilation_config = SimpleNamespace(cudagraph_mode=cudagraph_mode) + runner.cudagraph_manager = _FakeCudaGraphManager( + needs_capture, num_full_descs, piecewise_only + ) + runner.vllm_config = SimpleNamespace() + + events: list[str] = [] + runner.events = events + + def _capture_model() -> int: + events.append("capture") + # Simulate the manager's per-FULL-graph memory sampling. + samples = runner.cudagraph_manager._capture_mem_samples + if samples is not None: + samples.extend(mem_samples or []) + return captured_bytes + + runner.capture_model = _capture_model + return runner + + +def _patch_module(monkeypatch) -> None: + @contextlib.contextmanager + def _fake_set_current_vllm_config(_cfg): + yield + + monkeypatch.setattr(cgu, "set_current_vllm_config", _fake_set_current_vllm_config) + monkeypatch.setattr( + cgu, + "current_platform", + SimpleNamespace(graph_pool_handle=lambda: THROWAWAY_POOL), + ) + monkeypatch.setattr( + cgu, "_init_minimal_kv_cache_for_profiling", lambda r: r.events.append("init") + ) + monkeypatch.setattr( + cgu, "_teardown_profiling_state", lambda r: r.events.append("teardown") + ) + # The profiler reads free GPU memory before/after to compute what it + # retained; default to a constant (nothing retained). + monkeypatch.setattr(cgu.torch.accelerator, "empty_cache", lambda: None) + monkeypatch.setattr( + cgu.torch.accelerator, "get_memory_info", lambda: (1 << 30, 1 << 30) + ) + + +def test_profile_cudagraph_memory_disabled_returns_zero(monkeypatch): + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.NONE) + + result = cgu.profile_cudagraph_memory(runner) + + assert result == 0 + # No KV-cache bootstrap or teardown when cudagraphs are disabled. + assert runner.events == [] + + +def test_profile_cudagraph_memory_no_graphs_tears_down(monkeypatch): + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.FULL, needs_capture=False) + + result = cgu.profile_cudagraph_memory(runner) + + assert result == 0 + # Bootstrapped then cleaned up, without capturing or touching the pool. + assert runner.events == ["init", "teardown"] + assert runner.cudagraph_manager.pool == GLOBAL_POOL + + +def test_profile_cudagraph_memory_samples_and_extrapolates(monkeypatch): + _patch_module(monkeypatch) + gib = 1 << 30 + # Measured delta 1000 MiB includes the sampled FULL graphs (100 + 20 MiB). + # Extrapolated FULL cost for 3 graphs: 100 + 2 * 20 = 140 MiB. + runner = _make_profiling_runner( + CUDAGraphMode.FULL, + num_full_descs=3, + captured_bytes=1000 * gib, + mem_samples=[100 * gib, 20 * gib], + ) + + result = cgu.profile_cudagraph_memory(runner) + + assert result == (1000 - (100 + 20) + (100 + 2 * 20)) * gib + # Bootstrap, capture, and teardown run in order. + assert runner.events == ["init", "capture", "teardown"] + # Capture must use a throwaway pool, not the persistent global pool. + assert runner.cudagraph_manager.pool == THROWAWAY_POOL + # FULL capture must be limited to the largest few graphs. + assert ( + runner.cudagraph_manager._max_full_descs_to_capture + == cgu._FULL_GRAPH_PROFILING_SAMPLES + ) + + +def test_profile_cudagraph_memory_piecewise_only_returns_measured(monkeypatch): + _patch_module(monkeypatch) + captured_bytes = 5 << 30 + runner = _make_profiling_runner( + CUDAGraphMode.FULL_AND_PIECEWISE, + piecewise_only=True, + captured_bytes=captured_bytes, + ) + + result = cgu.profile_cudagraph_memory(runner) + + # No FULL graphs to sample or extrapolate: the measured delta is exact. + assert result == captured_bytes + + +def test_profile_cudagraph_memory_tears_down_on_capture_error(monkeypatch): + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.FULL) + + def _boom() -> int: + runner.events.append("capture") + raise RuntimeError("capture failed") + + runner.capture_model = _boom + + try: + cgu.profile_cudagraph_memory(runner) + except RuntimeError: + pass + else: + raise AssertionError("expected capture error to propagate") + + # Teardown still runs even if capture raises. + assert runner.events == ["init", "capture", "teardown"] + + +def test_profile_cudagraph_memory_restores_compilation_counters(monkeypatch): + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.FULL) + + def _capture_model() -> int: + compilation_counter.num_cudagraph_captured += 5 + compilation_counter.num_gpu_runner_capture_triggers += 1 + return 1 << 30 + + runner.capture_model = _capture_model + captured_before = compilation_counter.num_cudagraph_captured + triggers_before = compilation_counter.num_gpu_runner_capture_triggers + + cgu.profile_cudagraph_memory(runner) + + # Profiling captures are discarded, so they must not inflate the + # compilation counters; the real capture_model() runs later. + assert compilation_counter.num_cudagraph_captured == captured_before + assert compilation_counter.num_gpu_runner_capture_triggers == triggers_before + + +def test_model_runner_delegates_to_cudagraph_utils(monkeypatch): + runner = mrv2.GPUModelRunner.__new__(mrv2.GPUModelRunner) + monkeypatch.setattr(mrv2, "_profile_cudagraph_memory", lambda r: 42) + assert runner.profile_cudagraph_memory() == 42 + + +def test_extrapolate_full_graph_memory(): + mib = 1 << 20 + # No samples (e.g. no FULL graphs): nothing to add. + assert cgu._extrapolate_full_graph_memory([], 0) == 0 + # A single graph costs exactly its sample. + assert cgu._extrapolate_full_graph_memory([100 * mib], 1) == 100 * mib + # First capture + per-graph cost for the rest. + assert ( + cgu._extrapolate_full_graph_memory([100 * mib, 20 * mib], 5) + == (100 + 4 * 20) * mib + ) + # Per-graph cost is floored to account for driver overhead. + assert cgu._extrapolate_full_graph_memory([100 * mib, 0], 3) == (100 + 2 * 1) * mib + + +def test_profile_cudagraph_memory_clears_captured_graphs(monkeypatch): + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE) + + cleared: list[str] = [] + monkeypatch.setattr( + cgu.CUDAGraphWrapper, + "clear_all_graphs", + classmethod(lambda cls: cleared.append("piecewise")), + ) + monkeypatch.setattr( + cgu.BreakableCUDAGraphWrapper, + "clear_all_graphs", + classmethod(lambda cls: cleared.append("breakable")), + ) + + cgu.profile_cudagraph_memory(runner) + + # Profiling captures are discarded so the real capture re-captures them + # against the KV cache. + assert cleared == ["piecewise", "breakable"] + + +def test_profile_cudagraph_memory_redirects_wrapper_pools(monkeypatch): + """Piecewise wrappers must capture into the throwaway pool too. + + Profiling graphs captured into the persistent global pool and then + discarded drop the pool's use_count to 0, tripping the c10 allocator's + create_or_incref_pool assert when the real capture reuses that pool. + """ + _patch_module(monkeypatch) + runner = _make_profiling_runner(CUDAGraphMode.FULL_AND_PIECEWISE) + + class _FakeWrapper: + def __init__(self) -> None: + self.graph_pool: Any = GLOBAL_POOL + self.pool_during_capture: Any = None + + def clear_graphs(self) -> None: + pass + + wrapper = _FakeWrapper() + cgu.CUDAGraphWrapper._all_instances.add(wrapper) + try: + capture_model = runner.capture_model + + def _capture_model() -> int: + wrapper.pool_during_capture = wrapper.graph_pool + return capture_model() + + runner.capture_model = _capture_model + + cgu.profile_cudagraph_memory(runner) + + assert wrapper.pool_during_capture == THROWAWAY_POOL + assert wrapper.graph_pool == GLOBAL_POOL + finally: + cgu.CUDAGraphWrapper._all_instances.discard(wrapper) diff --git a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py index cf0a17720f..bf81cdd75f 100644 --- a/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/kimi_gdn_linear_attn.py @@ -416,7 +416,10 @@ class KimiGatedDeltaNetAttention(GatedDeltaNetAttention): ) assert isinstance(attn_metadata_raw, dict) - attn_metadata_narrowed = attn_metadata_raw[self.prefix] + attn_metadata_narrowed = attn_metadata_raw.get(self.prefix) + if attn_metadata_narrowed is None: + # Profile/warmup dummy runs skip mamba-family metadata. + return assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata) m = attn_metadata_narrowed has_initial_state = m.has_initial_state diff --git a/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py index a72707c7a1..2c1deec687 100644 --- a/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/olmo_gdn_linear_attn.py @@ -305,7 +305,10 @@ class OlmoHybridGatedDeltaNetAttention(GatedDeltaNetAttention): return assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] # type: ignore[assignment] + attn_metadata = attn_metadata.get(self.prefix) # type: ignore[assignment] + if attn_metadata is None: + # Profile/warmup dummy runs skip mamba-family metadata. + return assert isinstance(attn_metadata, GDNAttentionMetadata) has_initial_state = attn_metadata.has_initial_state spec_query_start_loc = attn_metadata.spec_query_start_loc diff --git a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py index d83b076509..743752b296 100644 --- a/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @@ -1202,13 +1202,14 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): forward_context = get_forward_context() attn_metadata_raw = forward_context.attn_metadata - if attn_metadata_raw is None: + attn_metadata = None + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is None: v_dim = core_attn_out.shape[-1] * core_attn_out.shape[-2] self._warmup_prefill_kernels(qkvz, v_dim) return - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] assert isinstance(attn_metadata, GDNAttentionMetadata) # The AITER fused reshape/conv kernel expects Qwen3-Next's interleaved @@ -1259,12 +1260,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): forward_context = get_forward_context() attn_metadata_raw = forward_context.attn_metadata - if attn_metadata_raw is None: + attn_metadata = None + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is None: self._warmup_prefill_kernels(mixed_qkv, 0) return - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] assert isinstance(attn_metadata, GDNAttentionMetadata) if ( @@ -1773,12 +1775,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): forward_context = get_forward_context() attn_metadata_raw = forward_context.attn_metadata qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size - if attn_metadata_raw is None: + attn_metadata = None + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is None: self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0) return - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] assert isinstance(attn_metadata, GDNAttentionMetadata) mixed_qkv, output_gate_flat = mixed_qkvz.split( [qkv_size, self.value_dim // self.tp_size], dim=-1 @@ -1856,12 +1859,13 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention): ) -> None: forward_context = get_forward_context() attn_metadata_raw = forward_context.attn_metadata - if attn_metadata_raw is None: + attn_metadata = None + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is None: self._warmup_prefill_kernels(mixed_qkv, 0) return - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] assert isinstance(attn_metadata, GDNAttentionMetadata) if ( self._can_use_fused_gdn_mtp_decode(attn_metadata) diff --git a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py index b69595e340..7f3b3f8dba 100644 --- a/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear/bailing_linear_attn.py @@ -542,7 +542,8 @@ class BailingMoELinearAttention(LinearAttention): attn_metadata = forward_context.attn_metadata if attn_metadata is not None: assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] # type: ignore + attn_metadata = attn_metadata.get(self.prefix) # type: ignore + if attn_metadata is not None: assert isinstance(attn_metadata, LinearAttentionMetadata) num_actual_tokens = ( attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens diff --git a/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py b/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py index 14c7d3d5f0..3842e3e2cd 100644 --- a/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear/minimax_linear_attn.py @@ -282,7 +282,8 @@ class MiniMaxText01LinearAttention(LinearAttention): attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is not None: assert isinstance(attn_metadata, LinearAttentionMetadata) num_actual_tokens = ( attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 1d3159d1e7..42f060e816 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -269,7 +269,8 @@ class MambaMixer(MambaBase, PluggableLayer): attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is not None: assert isinstance(attn_metadata, Mamba1AttentionMetadata) query_start_loc_p = attn_metadata.query_start_loc_p state_indices_tensor_p = attn_metadata.state_indices_tensor_p diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index 14b1121389..513402a0b1 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -710,7 +710,8 @@ class MambaMixer2(MambaBase, PluggableLayer): attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is not None: assert isinstance(attn_metadata, Mamba2AttentionMetadata) # conv_state must be (..., dim, width-1) for the conv kernels. # DS layout stores it that way directly; SD layout needs a diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py index de2efea50f..3f152623bd 100644 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ b/vllm/model_executor/layers/mamba/short_conv.py @@ -113,8 +113,10 @@ class ShortConv(MambaBase, PluggableLayer): attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] - assert isinstance(attn_metadata, ShortConvAttentionMetadata) + attn_metadata = attn_metadata_raw.get(self.prefix) + assert attn_metadata is None or isinstance( + attn_metadata, ShortConvAttentionMetadata + ) BCx, _ = self.in_proj(hidden_states) B, C, x = BCx.chunk(3, dim=-1) @@ -225,7 +227,8 @@ class ShortConv(MambaBase, PluggableLayer): attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] + attn_metadata = attn_metadata_raw.get(self.prefix) + if attn_metadata is not None: assert isinstance(attn_metadata, ShortConvAttentionMetadata) conv_state = ( self.kv_cache[0] diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 8e24a0ee02..e09a1756a5 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import gc from collections import defaultdict from collections.abc import Callable from dataclasses import dataclass from itertools import product -from typing import Any, NamedTuple, Protocol +from typing import TYPE_CHECKING, Any, NamedTuple, Protocol import torch import torch.nn as nn @@ -15,7 +16,8 @@ from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, ) from vllm.compilation.counter import compilation_counter -from vllm.config import VllmConfig +from vllm.compilation.cuda_graph import CUDAGraphWrapper +from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.compilation import CUDAGraphMode from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id from vllm.distributed.parallel_state import ( @@ -37,6 +39,9 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.interface import ModelState from vllm.v1.worker.utils import AttentionGroup +if TYPE_CHECKING: + from vllm.v1.worker.gpu.model_runner import GPUModelRunner + logger = init_logger(__name__) @@ -132,6 +137,12 @@ class CudaGraphManager: self._graphs_captured = False + # Profiling hooks, set only by profile_cudagraph_memory() below: cap + # FULL-mode capture at the N largest descriptors and record each + # captured FULL graph's memory delta for extrapolation. + self._max_full_descs_to_capture: int | None = None + self._capture_mem_samples: list[int] | None = None + self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {} self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {} @@ -327,6 +338,14 @@ class CudaGraphManager: continue descs = self._capture_descs[mode] + if ( + mode == CUDAGraphMode.FULL + and self._max_full_descs_to_capture is not None + ): + # Profiling only: capture a sample of the largest FULL + # graphs; the total cost is extrapolated from their + # per-graph memory deltas. + descs = descs[: self._max_full_descs_to_capture] if is_global_first_rank(): descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})") for desc in descs: @@ -362,6 +381,9 @@ class CudaGraphManager: set_graph_pool_id(self.pool) else: set_graph_pool_id(current_platform.graph_pool_handle()) + if self._capture_mem_samples is not None: + torch.accelerator.synchronize() + free_before = torch.accelerator.get_memory_info()[0] with torch.cuda.graph(graph, self.pool): forward_fn(CUDAGraphMode.NONE) # Join offloader's copy stream after forward to avoid @@ -369,6 +391,10 @@ class CudaGraphManager: # forks copy_stream, but wait_prefetch only happens in # the next forward pass. get_offloader().join_after_forward() + if self._capture_mem_samples is not None: + torch.accelerator.synchronize() + free_after = torch.accelerator.get_memory_info()[0] + self._capture_mem_samples.append(free_before - free_after) self.graphs[desc] = graph compilation_counter.num_cudagraph_captured += 1 self._graphs_captured = True @@ -668,3 +694,162 @@ def prepare_inputs_to_capture( for_capture=full_cudagraph, ) return AttentionState(attn_metadata, slot_mappings_by_layer) + + +# --------------------------------------------------------------------------- +# CUDA graph memory profiling +# --------------------------------------------------------------------------- + +# Number of FULL graphs captured during profiling; the total FULL capture +# cost is extrapolated from this sample to avoid a second full capture. +_FULL_GRAPH_PROFILING_SAMPLES = 2 +# Floor for the extrapolated per-graph cost (driver overhead per graph). +_MIN_PER_GRAPH_BYTES = 1 << 20 + + +@torch.inference_mode() +def profile_cudagraph_memory(runner: "GPUModelRunner") -> int: + """Estimate the GPU memory needed for CUDA graph capture. + + Called during memory profiling, *before* the real KV cache is allocated, + so that ``Worker.determine_available_memory`` can reserve headroom for + graph capture. Bootstraps a minimal KV cache, runs ``capture_model()`` + once, then releases everything so the real init/capture path starts clean. + + FULL graphs bake in KV cache pointers, so only the largest few are + captured (into a throwaway pool) and their total cost is extrapolated. + PIECEWISE, encoder and speculator graphs are measured in full. All + profiling captures are discarded afterwards: replaying graphs recorded + against the throwaway profiling state is unsafe (e.g. inductor graph + partition reclaims the storages of earlier cudagraph recordings once the + real capture records new ones, leading to use-after-free crashes). + """ + if runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE: + return 0 + + gc.collect() + torch.accelerator.empty_cache() + + with set_current_vllm_config(runner.vllm_config): + _init_minimal_kv_cache_for_profiling(runner) + + manager = runner.cudagraph_manager + assert manager is not None + + # Don't count profiling captures; the real capture_model() runs later. + saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured + saved_capture_triggers = compilation_counter.num_gpu_runner_capture_triggers + all_wrappers: list[Any] = [] + original_pools: dict[int, Any] = {} + try: + if not manager.needs_capture(): + return 0 + # Capture all profiling graphs into a throwaway pool so their memory + # is reclaimed on teardown rather than retained by the persistent + # global pool (which the real capture reuses). This must include the + # piecewise wrappers, not just the FULL-graph manager pool: graphs + # captured into the global pool and then discarded drop its use_count + # to 0, and the real capture on the same pool trips the c10 allocator's + # create_or_incref_pool assert ("use_count > 0 INTERNAL ASSERT FAILED"). + manager.pool = current_platform.graph_pool_handle() + if manager.use_breakable_cg: + # The breakable runner is otherwise created lazily during capture, + # after the pool swap below, and would capture into the global + # pool. Create it now so its pool gets swapped too. + manager.init_breakable_cg_runner(runner.model) + all_wrappers = list(CUDAGraphWrapper._all_instances) + list( + BreakableCUDAGraphWrapper._all_instances + ) + for wrapper in all_wrappers: + original_pools[id(wrapper)] = wrapper.graph_pool + wrapper.graph_pool = manager.pool + manager._max_full_descs_to_capture = _FULL_GRAPH_PROFILING_SAMPLES + mem_samples: list[int] = [] + manager._capture_mem_samples = mem_samples + + measured = int(runner.capture_model()) + + # The measured delta covers PIECEWISE, encoder and speculator graphs + # plus the sampled FULL graphs; swap the sampled FULL cost for the + # extrapolated total. FULL and PIECEWISE share one pool here just as + # they share the global pool at runtime, so the overlap is not + # double-counted. + num_full_graphs = len(manager._capture_descs.get(CUDAGraphMode.FULL, [])) + full_estimate = _extrapolate_full_graph_memory(mem_samples, num_full_graphs) + return max(measured - sum(mem_samples) + full_estimate, 0) + finally: + compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured + compilation_counter.num_gpu_runner_capture_triggers = saved_capture_triggers + CUDAGraphWrapper.clear_all_graphs() + BreakableCUDAGraphWrapper.clear_all_graphs() + for wrapper in all_wrappers: + if id(wrapper) in original_pools: + wrapper.graph_pool = original_pools[id(wrapper)] + _teardown_profiling_state(runner) + + +def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int: + """Extrapolate the total FULL capture cost from samples of the largest + graphs. The first capture allocates the pool baseline; later graphs mostly + reuse it, so the second sample is taken as the per-graph cost.""" + if not mem_samples: + return 0 + first_capture = mem_samples[0] + per_graph = max(mem_samples[1], _MIN_PER_GRAPH_BYTES) if len(mem_samples) > 1 else 0 + return first_capture + (total_graphs - 1) * per_graph + + +def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None: + """Allocate the smallest KV cache that still lets every graph be captured.""" + from vllm.v1.core.kv_cache_utils import ( + get_kv_cache_config_from_groups, + get_kv_cache_groups, + ) + + kv_cache_spec = runner.get_kv_cache_spec() + kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec) + # At least one block per sequence is required to capture the graphs. + min_blocks = ( + min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size) + or 1 + ) + saved_override = runner.cache_config.num_gpu_blocks_override + runner.cache_config.num_gpu_blocks_override = min_blocks + try: + minimal_config = get_kv_cache_config_from_groups( + runner.vllm_config, kv_cache_groups, available_memory=0 + ) + finally: + runner.cache_config.num_gpu_blocks_override = saved_override + + runner.initialize_kv_cache(minimal_config, is_profiling=True) + runner.cache_config.num_gpu_blocks = minimal_config.num_blocks + + +def _teardown_profiling_state(runner: "GPUModelRunner") -> None: + """Release the profiling KV cache and captured graphs while keeping model + weights, so the real ``initialize_kv_cache`` starts from a clean slate.""" + torch.accelerator.synchronize() + if hasattr(runner, "kv_caches"): + runner.kv_caches.clear() + if hasattr(runner, "attn_groups"): + runner.attn_groups.clear() + if hasattr(runner, "kv_cache_config"): + del runner.kv_cache_config + # Dropping the manager releases the profiling graphs and throwaway pool. + runner.cudagraph_manager = None + # Release encoder graphs captured during profiling; the real + # capture_model() re-captures them. + if runner.model_state.supports_mm_inputs: + runner.model_state.encoder_runner.clear() + # Detach profiling KV tensors held by attention layers. + for layer in runner.compilation_config.static_forward_context.values(): + if hasattr(layer, "kv_cache"): + kv_cache = layer.kv_cache + layer.kv_cache = ( + torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else [] + ) + runner.cache_config.num_gpu_blocks = None + runner.maybe_remove_all_loras(runner.lora_config) + gc.collect() + torch.accelerator.empty_cache() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 07e080a5bb..ce3de50016 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import ( BatchExecutionDescriptor, ModelCudaGraphManager, ) +from vllm.v1.worker.gpu.cudagraph_utils import ( + profile_cudagraph_memory as _profile_cudagraph_memory, +) from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp from vllm.v1.worker.gpu.ec_connector import get_ec_connector from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after @@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): def get_kv_cache_spec(self): return get_kv_cache_spec(self.vllm_config) - def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: + def initialize_kv_cache( + self, kv_cache_config: KVCacheConfig, is_profiling: bool = False + ) -> None: kv_cache_config = deepcopy(kv_cache_config) self.kv_cache_config = kv_cache_config @@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): tensor_parallel_size=self.parallel_config.tensor_parallel_size, kv_cache_config=self.kv_cache_config, max_num_reqs=self.max_num_reqs, + is_profiling=is_profiling, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, @@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.kernel_block_sizes, self.vllm_config, ) - self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict) + if is_profiling: + self.kv_connector = NO_OP_KV_CONNECTOR + else: + self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict) def _init_kv_zero_meta(self) -> None: """Build KV-block zeroing metadata; invoked from gpu_worker.""" @@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.pooling_runner is not None: self.pooling_runner.clear() + @torch.inference_mode() def profile_cudagraph_memory(self) -> int: - # NOTE(woosuk): It is TBD whether we keep this API or not. - return 0 + """Estimate the GPU memory required to capture CUDA graphs.""" + return _profile_cudagraph_memory(self) @torch.inference_mode() def capture_model(self) -> int: @@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin): slot_mappings, self.kv_cache_config ) assert block_tables is not None + attn_groups = self.attn_groups + if dummy_run and is_profile: + # Mamba layers take a cheap warmup path with no metadata; + # attention metadata is still built so those kernels tune. + attn_groups = [ + [g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)] + for groups in attn_groups + ] attn_metadata = self.model_state.prepare_attn( input_batch, batch_desc.cg_mode, block_tables, slot_mappings, - self.attn_groups, + attn_groups, self.kv_cache_config, # FULL replay reads capture-time metadata buffers. Re-stage them # from the zeroed dummy block tables instead of retaining state