[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
7e583e615c
commit
deb148e730
@@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
with patch.object(
|
||||
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
max_num_reqs, decode_kwargs = result
|
||||
assert max_num_reqs == expected_num_reqs
|
||||
expected_kwargs = {
|
||||
**PREFILL_KWARGS,
|
||||
"num_tokens": expected_num_reqs * query_len,
|
||||
@@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
force_attention=True,
|
||||
profile_seq_lens=query_len + 1,
|
||||
)
|
||||
assert result == (expected_num_reqs, expected_kwargs)
|
||||
assert decode_kwargs == expected_kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
(dict(use_replayssm=False), True),
|
||||
(dict(backend=MambaBackendEnum.TRITON), True),
|
||||
({}, False),
|
||||
(dict(num_blocks=1), True),
|
||||
],
|
||||
ids=[
|
||||
"replayssm_disabled",
|
||||
"non_flashinfer_backend",
|
||||
"kernel_unavailable",
|
||||
"zero_non_padding_slots",
|
||||
],
|
||||
ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"],
|
||||
)
|
||||
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
with patch.object(
|
||||
@@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
"flashinfer_replayssm_autotune_supported",
|
||||
return_value=flashinfer_supported,
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers():
|
||||
)
|
||||
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
|
||||
tracked = (
|
||||
*mixer.kv_cache,
|
||||
mixer._replayssm_ring_start,
|
||||
mixer._replayssm_prev_num_accepted,
|
||||
mixer._replayssm_prev_query_len,
|
||||
)
|
||||
|
||||
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)
|
||||
|
||||
Reference in New Issue
Block a user