[V1] [Hybrid] Refactor mamba state shape calculation; enable V1 via cli (#20840)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
This commit is contained in:
Thomas Parnell
2025-07-15 04:04:35 -07:00
committed by GitHub
parent c586b55667
commit 3534c39a20
14 changed files with 441 additions and 353 deletions
+3 -55
View File
@@ -42,7 +42,7 @@ from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingType
from vllm.sequence import IntermediateTensors
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
GiB_bytes, LazyLoader, async_tensor_h2d, cdiv,
GiB_bytes, LazyLoader, async_tensor_h2d,
check_use_alibi, get_dtype_size,
is_pin_memory_available, round_up)
from vllm.v1.attention.backends.mamba_attn import Mamba2AttentionBackend
@@ -2648,9 +2648,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
"Prefix caching is not supported for Mamba yet.")
max_model_len = self.vllm_config.model_config.max_model_len
page_size_padded = self._maybe_pad_mamba_page_size(
attn_layers, mamba_layers, kv_cache_spec, max_model_len,
block_size)
page_size_padded = (
self.vllm_config.cache_config.mamba_page_size_padded)
# Set block_size to max_model_len, so that mamba model will always
# have only one block in the KV cache.
@@ -2662,54 +2661,3 @@ class GPUModelRunner(LoRAModelRunnerMixin):
page_size_padded=page_size_padded)
return kv_cache_spec
def _maybe_pad_mamba_page_size(
self,
attn_layers: dict[str, Attention],
mamba_layers: dict[str, MambaBase],
kv_cache_spec: dict[str, KVCacheSpec],
max_model_len: int,
block_size: int,
) -> Optional[int]:
"""
Ensure that page size of attention KV cache groups is greater than or
equal to the mamba KV cache groups. If not, we suggest to the user
how to set the attention block size to ensure that it is.
If the attention page size is strictly greater than the mamba page size,
we pad the mamba page size to make them equal.
Args:
attn_layers: Attention layers
mamba_layers: Mamba layers
kv_cache_spec: KV cache spec (populated with attention layers)
Returns:
Optional[int]: Mamba page size with padding (None if no padding).
"""
if len(attn_layers) == 0:
return None
attn_layer_name = next(iter(attn_layers))
attn_page_size = kv_cache_spec[attn_layer_name].page_size_bytes
mamba_layer_name = next(iter(mamba_layers))
mamba_page_size = MambaSpec(
shapes=mamba_layers[mamba_layer_name].get_state_shape(),
dtype=self.kv_cache_dtype,
block_size=max_model_len).page_size_bytes
if attn_page_size < mamba_page_size:
# attention page size (for 16 tokens)
attn_page_size_16 = 16 * attn_page_size // block_size
# some attention backends (e.g. FA) only support setting
# block size to multiple of 16, so let's suggest a value
# that would work (note: FA is currently not compatible
# with mamba layers, use FlashInfer instead).
suggest_attn_block_size = 16 * cdiv(mamba_page_size,
attn_page_size_16)
raise ValueError(
"Attention block size should be increased to at least "
f"{suggest_attn_block_size} in order to match "
"the mamba page size")
return attn_page_size