[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
+6
View File
@@ -13,6 +13,7 @@ import torch
from vllm.config import CacheConfig, VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.mamba.mamba_mixer2 import share_replayssm_ring_trackers
from vllm.model_executor.layers.utils import warmup_rocm_skinny_gemm_workspaces
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
from vllm.model_executor.models.utils import extract_layer_index
@@ -575,6 +576,7 @@ def bind_kv_cache(
forward_context: dict[str, Attention],
runner_kv_caches: list[torch.Tensor],
num_attn_module: int = 1,
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
) -> None:
"""
Bind the allocated KV cache to both ModelRunner and forward context so
@@ -600,6 +602,7 @@ def bind_kv_cache(
for layer_name in kv_caches:
index2name[extract_layer_index(layer_name, num_attn_module)].append(layer_name)
ordered_layer_names: list[str] = []
for layer_index in sorted(index2name.keys()):
layer_names = index2name[layer_index]
if len(layer_names) > 1:
@@ -613,6 +616,7 @@ def bind_kv_cache(
current_platform.check_runner_kv_caches_multi_layer()
for layer_name in layer_names:
runner_kv_caches.append(kv_caches[layer_name])
ordered_layer_names.append(layer_name)
# Bind kv_caches to forward context. Each layer's bind_kv_cache unpacks
# its raw allocation into the per-layer view(s) it needs (e.g. Mamba
@@ -621,6 +625,8 @@ def bind_kv_cache(
for layer_name, kv_cache in kv_caches.items():
forward_context[layer_name].bind_kv_cache(kv_cache)
share_replayssm_ring_trackers(ordered_layer_names, forward_context, kv_cache_groups)
def clear_layer_kv_caches(layers: Iterable[Any]) -> None:
"""Detach the KV/state cache tensors installed by bind_kv_cache().