[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -7,12 +7,15 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
FlashInferSSUBackend,
|
||||
TritonSSUBackend,
|
||||
get_mamba_ssu_backend,
|
||||
initialize_mamba_ssu_backend,
|
||||
reset_replayssm_ring_trackers,
|
||||
selective_state_update,
|
||||
update_replayssm_ring_trackers,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
@@ -30,6 +33,47 @@ except ImportError:
|
||||
HAS_FLASHINFER = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_backend_state():
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
old_backend = mod._mamba_ssu_backend
|
||||
old_replayssm_kernel = mod._flashinfer_replayssm_kernel
|
||||
yield
|
||||
mod._mamba_ssu_backend = old_backend
|
||||
mod._flashinfer_replayssm_kernel = old_replayssm_kernel
|
||||
|
||||
|
||||
def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for _ in range(33):
|
||||
update_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
state_batch_indices,
|
||||
logical_window=16,
|
||||
ring_buffer_len=17,
|
||||
)
|
||||
observed.append((int(ring_start[1]), int(prev_num_accepted[1])))
|
||||
|
||||
assert observed[4] == (0, 5)
|
||||
assert observed[15] == (0, 16)
|
||||
assert observed[16] == (16, 1)
|
||||
assert observed[31] == (16, 16)
|
||||
assert observed[32] == (15, 1)
|
||||
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
|
||||
|
||||
|
||||
def _kv_cache_config_with_ssu(
|
||||
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2,
|
||||
) -> KVCacheConfig:
|
||||
@@ -116,11 +160,10 @@ def test_flashinfer_forwards_ssu_algorithm(
|
||||
def test_uninitialized_backend_raises():
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
old = mod._mamba_ssu_backend
|
||||
# restore_backend_state (autouse) puts the global back afterwards.
|
||||
mod._mamba_ssu_backend = None
|
||||
with pytest.raises(RuntimeError, match="not been initialized"):
|
||||
get_mamba_ssu_backend()
|
||||
mod._mamba_ssu_backend = old
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -186,3 +229,28 @@ def test_triton_basic_call():
|
||||
out=out,
|
||||
)
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_ring_len"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 17),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
base_shapes = ((64, 3), (8, 4, 16))
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
base_shapes,
|
||||
n_groups=4,
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
assert shapes[2:] == (
|
||||
(8, expected_ring_len, 4),
|
||||
(8, expected_ring_len),
|
||||
(2, expected_ring_len, 16),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user