[Model Runner V2] Reserve CUDA graph memory (#53306)

Signed-off-by: Anh Tran <anh.tran.3889@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Anh Tran <anh.tran.3889@gmail.com>
Co-authored-by: anhtra3889 <anhtra@nvidia.com>
This commit is contained in:
Nick Hill
2026-08-23 23:11:19 -07:00
committed by GitHub
co-authored by Anh Tran anhtra3889
parent 8c2bbe00d5
commit a4d70bef37
13 changed files with 573 additions and 31 deletions
+187 -2
View File
@@ -1,10 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from collections import defaultdict
from collections.abc import Callable
from dataclasses import dataclass
from itertools import product
from typing import Any, NamedTuple, Protocol
from typing import TYPE_CHECKING, Any, NamedTuple, Protocol
import torch
import torch.nn as nn
@@ -15,7 +16,8 @@ from vllm.compilation.breakable_cudagraph import (
is_breakable_cudagraph_enabled,
)
from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig
from vllm.compilation.cuda_graph import CUDAGraphWrapper
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.device_communicators.pynccl_allocator import set_graph_pool_id
from vllm.distributed.parallel_state import (
@@ -37,6 +39,9 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.utils import AttentionGroup
if TYPE_CHECKING:
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
logger = init_logger(__name__)
@@ -132,6 +137,12 @@ class CudaGraphManager:
self._graphs_captured = False
# Profiling hooks, set only by profile_cudagraph_memory() below: cap
# FULL-mode capture at the N largest descriptors and record each
# captured FULL graph's memory delta for extrapolation.
self._max_full_descs_to_capture: int | None = None
self._capture_mem_samples: list[int] | None = None
self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {}
self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {}
@@ -327,6 +338,14 @@ class CudaGraphManager:
continue
descs = self._capture_descs[mode]
if (
mode == CUDAGraphMode.FULL
and self._max_full_descs_to_capture is not None
):
# Profiling only: capture a sample of the largest FULL
# graphs; the total cost is extrapolated from their
# per-graph memory deltas.
descs = descs[: self._max_full_descs_to_capture]
if is_global_first_rank():
descs = tqdm(descs, desc=f"{progress_bar_desc} ({mode.name})")
for desc in descs:
@@ -362,6 +381,9 @@ class CudaGraphManager:
set_graph_pool_id(self.pool)
else:
set_graph_pool_id(current_platform.graph_pool_handle())
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_before = torch.accelerator.get_memory_info()[0]
with torch.cuda.graph(graph, self.pool):
forward_fn(CUDAGraphMode.NONE)
# Join offloader's copy stream after forward to avoid
@@ -369,6 +391,10 @@ class CudaGraphManager:
# forks copy_stream, but wait_prefetch only happens in
# the next forward pass.
get_offloader().join_after_forward()
if self._capture_mem_samples is not None:
torch.accelerator.synchronize()
free_after = torch.accelerator.get_memory_info()[0]
self._capture_mem_samples.append(free_before - free_after)
self.graphs[desc] = graph
compilation_counter.num_cudagraph_captured += 1
self._graphs_captured = True
@@ -668,3 +694,162 @@ def prepare_inputs_to_capture(
for_capture=full_cudagraph,
)
return AttentionState(attn_metadata, slot_mappings_by_layer)
# ---------------------------------------------------------------------------
# CUDA graph memory profiling
# ---------------------------------------------------------------------------
# Number of FULL graphs captured during profiling; the total FULL capture
# cost is extrapolated from this sample to avoid a second full capture.
_FULL_GRAPH_PROFILING_SAMPLES = 2
# Floor for the extrapolated per-graph cost (driver overhead per graph).
_MIN_PER_GRAPH_BYTES = 1 << 20
@torch.inference_mode()
def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
"""Estimate the GPU memory needed for CUDA graph capture.
Called during memory profiling, *before* the real KV cache is allocated,
so that ``Worker.determine_available_memory`` can reserve headroom for
graph capture. Bootstraps a minimal KV cache, runs ``capture_model()``
once, then releases everything so the real init/capture path starts clean.
FULL graphs bake in KV cache pointers, so only the largest few are
captured (into a throwaway pool) and their total cost is extrapolated.
PIECEWISE, encoder and speculator graphs are measured in full. All
profiling captures are discarded afterwards: replaying graphs recorded
against the throwaway profiling state is unsafe (e.g. inductor graph
partition reclaims the storages of earlier cudagraph recordings once the
real capture records new ones, leading to use-after-free crashes).
"""
if runner.compilation_config.cudagraph_mode == CUDAGraphMode.NONE:
return 0
gc.collect()
torch.accelerator.empty_cache()
with set_current_vllm_config(runner.vllm_config):
_init_minimal_kv_cache_for_profiling(runner)
manager = runner.cudagraph_manager
assert manager is not None
# Don't count profiling captures; the real capture_model() runs later.
saved_num_cudagraph_captured = compilation_counter.num_cudagraph_captured
saved_capture_triggers = compilation_counter.num_gpu_runner_capture_triggers
all_wrappers: list[Any] = []
original_pools: dict[int, Any] = {}
try:
if not manager.needs_capture():
return 0
# Capture all profiling graphs into a throwaway pool so their memory
# is reclaimed on teardown rather than retained by the persistent
# global pool (which the real capture reuses). This must include the
# piecewise wrappers, not just the FULL-graph manager pool: graphs
# captured into the global pool and then discarded drop its use_count
# to 0, and the real capture on the same pool trips the c10 allocator's
# create_or_incref_pool assert ("use_count > 0 INTERNAL ASSERT FAILED").
manager.pool = current_platform.graph_pool_handle()
if manager.use_breakable_cg:
# The breakable runner is otherwise created lazily during capture,
# after the pool swap below, and would capture into the global
# pool. Create it now so its pool gets swapped too.
manager.init_breakable_cg_runner(runner.model)
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
)
for wrapper in all_wrappers:
original_pools[id(wrapper)] = wrapper.graph_pool
wrapper.graph_pool = manager.pool
manager._max_full_descs_to_capture = _FULL_GRAPH_PROFILING_SAMPLES
mem_samples: list[int] = []
manager._capture_mem_samples = mem_samples
measured = int(runner.capture_model())
# The measured delta covers PIECEWISE, encoder and speculator graphs
# plus the sampled FULL graphs; swap the sampled FULL cost for the
# extrapolated total. FULL and PIECEWISE share one pool here just as
# they share the global pool at runtime, so the overlap is not
# double-counted.
num_full_graphs = len(manager._capture_descs.get(CUDAGraphMode.FULL, []))
full_estimate = _extrapolate_full_graph_memory(mem_samples, num_full_graphs)
return max(measured - sum(mem_samples) + full_estimate, 0)
finally:
compilation_counter.num_cudagraph_captured = saved_num_cudagraph_captured
compilation_counter.num_gpu_runner_capture_triggers = saved_capture_triggers
CUDAGraphWrapper.clear_all_graphs()
BreakableCUDAGraphWrapper.clear_all_graphs()
for wrapper in all_wrappers:
if id(wrapper) in original_pools:
wrapper.graph_pool = original_pools[id(wrapper)]
_teardown_profiling_state(runner)
def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int:
"""Extrapolate the total FULL capture cost from samples of the largest
graphs. The first capture allocates the pool baseline; later graphs mostly
reuse it, so the second sample is taken as the per-graph cost."""
if not mem_samples:
return 0
first_capture = mem_samples[0]
per_graph = max(mem_samples[1], _MIN_PER_GRAPH_BYTES) if len(mem_samples) > 1 else 0
return first_capture + (total_graphs - 1) * per_graph
def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
"""Allocate the smallest KV cache that still lets every graph be captured."""
from vllm.v1.core.kv_cache_utils import (
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
kv_cache_spec = runner.get_kv_cache_spec()
kv_cache_groups = get_kv_cache_groups(runner.vllm_config, kv_cache_spec)
# At least one block per sequence is required to capture the graphs.
min_blocks = (
min(runner.max_num_reqs, runner.compilation_config.max_cudagraph_capture_size)
or 1
)
saved_override = runner.cache_config.num_gpu_blocks_override
runner.cache_config.num_gpu_blocks_override = min_blocks
try:
minimal_config = get_kv_cache_config_from_groups(
runner.vllm_config, kv_cache_groups, available_memory=0
)
finally:
runner.cache_config.num_gpu_blocks_override = saved_override
runner.initialize_kv_cache(minimal_config, is_profiling=True)
runner.cache_config.num_gpu_blocks = minimal_config.num_blocks
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
"""Release the profiling KV cache and captured graphs while keeping model
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
torch.accelerator.synchronize()
if hasattr(runner, "kv_caches"):
runner.kv_caches.clear()
if hasattr(runner, "attn_groups"):
runner.attn_groups.clear()
if hasattr(runner, "kv_cache_config"):
del runner.kv_cache_config
# Dropping the manager releases the profiling graphs and throwaway pool.
runner.cudagraph_manager = None
# Release encoder graphs captured during profiling; the real
# capture_model() re-captures them.
if runner.model_state.supports_mm_inputs:
runner.model_state.encoder_runner.clear()
# Detach profiling KV tensors held by attention layers.
for layer in runner.compilation_config.static_forward_context.values():
if hasattr(layer, "kv_cache"):
kv_cache = layer.kv_cache
layer.kv_cache = (
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
)
runner.cache_config.num_gpu_blocks = None
runner.maybe_remove_all_loras(runner.lora_config)
gc.collect()
torch.accelerator.empty_cache()
+23 -5
View File
@@ -94,6 +94,9 @@ from vllm.v1.worker.gpu.cudagraph_utils import (
BatchExecutionDescriptor,
ModelCudaGraphManager,
)
from vllm.v1.worker.gpu.cudagraph_utils import (
profile_cudagraph_memory as _profile_cudagraph_memory,
)
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.ec_connector import get_ec_connector
from vllm.v1.worker.gpu.eplb_utils import EPLBController, step_eplb_after
@@ -499,7 +502,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def get_kv_cache_spec(self):
return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
def initialize_kv_cache(
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
@@ -585,6 +590,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
tensor_parallel_size=self.parallel_config.tensor_parallel_size,
kv_cache_config=self.kv_cache_config,
max_num_reqs=self.max_num_reqs,
is_profiling=is_profiling,
)
self.cudagraph_manager = ModelCudaGraphManager(
self.vllm_config,
@@ -618,7 +624,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.kernel_block_sizes,
self.vllm_config,
)
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
else:
self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict)
def _init_kv_zero_meta(self) -> None:
"""Build KV-block zeroing metadata; invoked from gpu_worker."""
@@ -832,9 +841,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pooling_runner is not None:
self.pooling_runner.clear()
@torch.inference_mode()
def profile_cudagraph_memory(self) -> int:
# NOTE(woosuk): It is TBD whether we keep this API or not.
return 0
"""Estimate the GPU memory required to capture CUDA graphs."""
return _profile_cudagraph_memory(self)
@torch.inference_mode()
def capture_model(self) -> int:
@@ -1521,12 +1531,20 @@ class GPUModelRunner(LoRAModelRunnerMixin):
slot_mappings, self.kv_cache_config
)
assert block_tables is not None
attn_groups = self.attn_groups
if dummy_run and is_profile:
# Mamba layers take a cheap warmup path with no metadata;
# attention metadata is still built so those kernels tune.
attn_groups = [
[g for g in groups if not isinstance(g.kv_cache_spec, MambaSpec)]
for groups in attn_groups
]
attn_metadata = self.model_state.prepare_attn(
input_batch,
batch_desc.cg_mode,
block_tables,
slot_mappings,
self.attn_groups,
attn_groups,
self.kv_cache_config,
# FULL replay reads capture-time metadata buffers. Re-stage them
# from the zeroed dummy block tables instead of retaining state