[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-30 21:41:17 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent 7e583e615c
commit deb148e730
18 changed files with 1029 additions and 113 deletions
+19 -4
View File
@@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
with patch.object(
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
):
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is not None
max_num_reqs, decode_kwargs = result
assert max_num_reqs == expected_num_reqs
expected_kwargs = {
**PREFILL_KWARGS,
"num_tokens": expected_num_reqs * query_len,
@@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
force_attention=True,
profile_seq_lens=query_len + 1,
)
assert result == (expected_num_reqs, expected_kwargs)
assert decode_kwargs == expected_kwargs
@pytest.mark.parametrize(
@@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
(dict(use_replayssm=False), True),
(dict(backend=MambaBackendEnum.TRITON), True),
({}, False),
(dict(num_blocks=1), True),
],
ids=[
"replayssm_disabled",
"non_flashinfer_backend",
"kernel_unavailable",
"zero_non_padding_slots",
],
ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"],
)
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
with patch.object(
@@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
"flashinfer_replayssm_autotune_supported",
return_value=flashinfer_supported,
):
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is None
@@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers():
)
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
tracked = (
*mixer.kv_cache,
mixer._replayssm_ring_start,
mixer._replayssm_prev_num_accepted,
mixer._replayssm_prev_query_len,
)
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)