[Mamba][Bugfix] Raise on insufficient cache blocks instead of silently capping cudagraph sizes (#38270)

Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-03-30 09:41:50 +00:00
committed by GitHub
parent 03ac6ca895
commit cc06b4e86b
4 changed files with 41 additions and 131 deletions
+34 -15
View File
@@ -5800,7 +5800,7 @@ class GPUModelRunner(
)
self.cache_config.num_gpu_blocks_override = saved_override
self.initialize_kv_cache(minimal_config)
self.initialize_kv_cache(minimal_config, is_profiling=True)
self.cache_config.num_gpu_blocks = minimal_config.num_blocks
logger.debug("Initialized minimal KV cache for CUDA graph profiling")
@@ -6121,7 +6121,11 @@ class GPUModelRunner(
torch.accelerator.synchronize()
self.maybe_remove_all_loras(self.lora_config)
def initialize_attn_backend(self, kv_cache_config: KVCacheConfig) -> None:
def initialize_attn_backend(
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
) -> None:
"""
Initialize the attention backends and attention metadata builders.
"""
@@ -6193,7 +6197,9 @@ class GPUModelRunner(
# Resolve cudagraph_mode before actually initialize metadata_builders
self._check_and_update_cudagraph_mode(
attention_backend_list, kv_cache_config.kv_cache_groups
attention_backend_list,
kv_cache_config.kv_cache_groups,
is_profiling=is_profiling,
)
# Check if attention backend supports PCP&DCP and related features.
@@ -6237,6 +6243,7 @@ class GPUModelRunner(
self,
attention_backends: list[set[type[AttentionBackend]]],
kv_cache_groups: list[KVCacheGroupSpec],
is_profiling: bool = False,
) -> None:
"""
Resolve the cudagraph_mode when there are multiple attention
@@ -6377,21 +6384,29 @@ class GPUModelRunner(
self.uniform_decode_query_len, self.parallel_config.tensor_parallel_size
)
# If the model has Mamba layers and cudagraph mode includes FULL
# decode, cap cudagraph capture sizes to the number of available
# Mamba cache blocks. Each decode request needs one conv_state
# cache line, so capture batch sizes cannot exceed num_blocks.
# Only FULL decode graphs are affected because PIECEWISE captures
# run GDN/Mamba ops eagerly (prefill path, no causal_conv1d_update).
# For Mamba models with FULL decode cudagraphs, each decode
# sequence needs one Mamba cache block. The decode cudagraph
# dispatcher already caps batch sizes at max_num_seqs, so we just
# need to verify that enough blocks exist. Raising here instead
# of silently capping cudagraph_capture_sizes avoids unintended
# restrictions on PIECEWISE (prefill) cudagraphs.
# See: https://github.com/vllm-project/vllm/issues/34094
if cudagraph_mode.has_full_cudagraphs():
if cudagraph_mode.has_full_cudagraphs() and not is_profiling:
has_mamba = any(
isinstance(g.kv_cache_spec, MambaSpec) for g in kv_cache_groups
)
if has_mamba and self.kv_cache_config is not None:
self.compilation_config.adjust_cudagraph_sizes_for_mamba_cache(
self.kv_cache_config.num_blocks
)
num_blocks = self.kv_cache_config.num_blocks
if self.max_num_reqs > num_blocks:
raise ValueError(
f"max_num_seqs ({self.max_num_reqs}) exceeds "
f"available Mamba cache blocks ({num_blocks}). "
f"Each decode sequence requires one Mamba cache "
f"block, so CUDA graph capture cannot proceed. "
f"Please lower max_num_seqs to at most "
f"{num_blocks} or increase "
f"gpu_memory_utilization."
)
# Trigger cudagraph dispatching keys initialization after
# resolved cudagraph mode.
@@ -6752,7 +6767,11 @@ class GPUModelRunner(
else:
break
def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
def initialize_kv_cache(
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
) -> None:
"""
Initialize KV cache based on `kv_cache_config`.
Args:
@@ -6764,7 +6783,7 @@ class GPUModelRunner(
self._mamba_copy_bufs = None
self.may_add_encoder_only_layers_to_kv_cache_config()
self.maybe_add_kv_sharing_layers_to_kv_cache_groups(kv_cache_config)
self.initialize_attn_backend(kv_cache_config)
self.initialize_attn_backend(kv_cache_config, is_profiling=is_profiling)
# The kernel block size for all KV cache groups. For example, if
# kv_cache_manager uses block_size 256 for a given group, but the attention
# backends for that group only supports block_size 64, we will return