[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -1,11 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.v1.worker.utils import bind_kv_cache
|
||||
|
||||
|
||||
class _TestReplaySSMMixer(MambaMixer2):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.use_replayssm = True
|
||||
self.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
|
||||
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
|
||||
self._updates_replayssm_trackers = True
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||
return ((2,), (3,), (4,), (5,), (6,))
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
return (torch.float32,) * 5
|
||||
|
||||
|
||||
def _packed_replayssm_cache(num_blocks: int) -> torch.Tensor:
|
||||
return torch.full((num_blocks, 1, 1, 80), 0, dtype=torch.int8)
|
||||
|
||||
|
||||
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
|
||||
mixers = [_TestReplaySSMMixer() for _ in range(3)]
|
||||
layer_names = [f"layers.{i}.mixer" for i in range(3)]
|
||||
ctx = dict(zip(layer_names, mixers))
|
||||
# Reverse insertion order: updater must follow layer index, not dict order.
|
||||
kv_cache = {
|
||||
layer_names[2]: _packed_replayssm_cache(4),
|
||||
layer_names[1]: _packed_replayssm_cache(4),
|
||||
layer_names[0]: _packed_replayssm_cache(4),
|
||||
}
|
||||
kv_cache_groups = [
|
||||
SimpleNamespace(layer_names=[layer_names[0], layer_names[2]]),
|
||||
SimpleNamespace(layer_names=[layer_names[1]]),
|
||||
]
|
||||
|
||||
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
|
||||
|
||||
assert (
|
||||
mixers[0]._replayssm_ring_start.data_ptr()
|
||||
== mixers[2]._replayssm_ring_start.data_ptr()
|
||||
)
|
||||
assert (
|
||||
mixers[0]._replayssm_prev_num_accepted.data_ptr()
|
||||
== mixers[2]._replayssm_prev_num_accepted.data_ptr()
|
||||
)
|
||||
assert (
|
||||
mixers[1]._replayssm_ring_start.data_ptr()
|
||||
!= mixers[0]._replayssm_ring_start.data_ptr()
|
||||
)
|
||||
# Group {0, 2} shares trackers; layer 2 (not 0) updates after both run.
|
||||
assert [m._updates_replayssm_trackers for m in mixers] == [False, True, True]
|
||||
|
||||
|
||||
def test_bind_kv_cache(default_vllm_config):
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
|
||||
|
||||
Reference in New Issue
Block a user