[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -13,6 +13,7 @@ import torch
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import share_replayssm_ring_trackers
|
||||
from vllm.model_executor.layers.utils import warmup_rocm_skinny_gemm_workspaces
|
||||
from vllm.model_executor.models.interfaces import MultiModalEmbeddings
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
@@ -575,6 +576,7 @@ def bind_kv_cache(
|
||||
forward_context: dict[str, Attention],
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
num_attn_module: int = 1,
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
@@ -600,6 +602,7 @@ def bind_kv_cache(
|
||||
for layer_name in kv_caches:
|
||||
index2name[extract_layer_index(layer_name, num_attn_module)].append(layer_name)
|
||||
|
||||
ordered_layer_names: list[str] = []
|
||||
for layer_index in sorted(index2name.keys()):
|
||||
layer_names = index2name[layer_index]
|
||||
if len(layer_names) > 1:
|
||||
@@ -613,6 +616,7 @@ def bind_kv_cache(
|
||||
current_platform.check_runner_kv_caches_multi_layer()
|
||||
for layer_name in layer_names:
|
||||
runner_kv_caches.append(kv_caches[layer_name])
|
||||
ordered_layer_names.append(layer_name)
|
||||
|
||||
# Bind kv_caches to forward context. Each layer's bind_kv_cache unpacks
|
||||
# its raw allocation into the per-layer view(s) it needs (e.g. Mamba
|
||||
@@ -621,6 +625,8 @@ def bind_kv_cache(
|
||||
for layer_name, kv_cache in kv_caches.items():
|
||||
forward_context[layer_name].bind_kv_cache(kv_cache)
|
||||
|
||||
share_replayssm_ring_trackers(ordered_layer_names, forward_context, kv_cache_groups)
|
||||
|
||||
|
||||
def clear_layer_kv_caches(layers: Iterable[Any]) -> None:
|
||||
"""Detach the KV/state cache tensors installed by bind_kv_cache().
|
||||
|
||||
Reference in New Issue
Block a user