[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
7e583e615c
commit
deb148e730
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
FlashInferSSUBackend,
|
||||
TritonSSUBackend,
|
||||
commit_replayssm_ring_trackers,
|
||||
get_mamba_ssu_backend,
|
||||
initialize_mamba_ssu_backend,
|
||||
reset_replayssm_ring_trackers,
|
||||
selective_state_update,
|
||||
selective_state_update_replayssm_flashinfer,
|
||||
update_replayssm_ring_trackers,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -32,6 +38,13 @@ try:
|
||||
except ImportError:
|
||||
HAS_FLASHINFER = False
|
||||
|
||||
try:
|
||||
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
|
||||
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
|
||||
except ImportError:
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_backend_state():
|
||||
@@ -47,6 +60,7 @@ def restore_backend_state():
|
||||
def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
update_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
logical_window=16,
|
||||
ring_buffer_len=17,
|
||||
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("accepted_sequence", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[4] * 22,
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 8, 4),
|
||||
(0, 12, 4),
|
||||
(0, 16, 4),
|
||||
(16, 4, 4),
|
||||
(16, 8, 4),
|
||||
(16, 12, 4),
|
||||
(16, 16, 4),
|
||||
(12, 4, 4),
|
||||
(12, 8, 4),
|
||||
(12, 12, 4),
|
||||
(12, 16, 4),
|
||||
(8, 4, 4),
|
||||
(8, 8, 4),
|
||||
(8, 12, 4),
|
||||
(8, 16, 4),
|
||||
(4, 4, 4),
|
||||
(4, 8, 4),
|
||||
(4, 12, 4),
|
||||
(4, 16, 4),
|
||||
(0, 4, 4),
|
||||
],
|
||||
id="all-accepted",
|
||||
),
|
||||
pytest.param(
|
||||
[4, 4, 0, 3, 4, 2, 4, 1],
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 4, 4),
|
||||
(0, 7, 4),
|
||||
(0, 11, 4),
|
||||
(0, 13, 4),
|
||||
(13, 4, 4),
|
||||
(13, 5, 4),
|
||||
],
|
||||
id="mixed",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
|
||||
logical_window = 16
|
||||
num_speculative_tokens = 3
|
||||
query_len = 1 + num_speculative_tokens
|
||||
ring_buffer_len = logical_window + 1 + num_speculative_tokens
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in accepted_sequence:
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window,
|
||||
ring_buffer_len,
|
||||
)
|
||||
snapshot = (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
)
|
||||
observed.append(snapshot)
|
||||
assert snapshot[1] + snapshot[2] <= ring_buffer_len
|
||||
|
||||
assert observed == expected
|
||||
|
||||
|
||||
def test_replayssm_resume_resets_commit_history():
|
||||
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([3], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 4)
|
||||
|
||||
|
||||
def test_replayssm_commit_tracker_ragged_query_lengths():
|
||||
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in ([4, 2], [3, 1]):
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
observed.append(
|
||||
[
|
||||
(
|
||||
ring_start[slot].item(),
|
||||
prev_num_accepted[slot].item(),
|
||||
prev_query_len[slot].item(),
|
||||
)
|
||||
for slot in (1, 2)
|
||||
]
|
||||
)
|
||||
|
||||
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["commit", "reset"])
|
||||
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
|
||||
num_states = 3
|
||||
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor(
|
||||
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
if operation == "commit":
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
expected_valid = (2, 5, 4)
|
||||
else:
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
expected_valid = (0, 0, 0)
|
||||
|
||||
assert (
|
||||
ring_start.tolist(),
|
||||
prev_num_accepted.tolist(),
|
||||
prev_query_len.tolist(),
|
||||
) == (
|
||||
[11, expected_valid[0], 33],
|
||||
[11, expected_valid[1], 33],
|
||||
[11, expected_valid[2], 33],
|
||||
)
|
||||
|
||||
|
||||
def _kv_cache_config_with_ssu(
|
||||
@@ -231,14 +442,190 @@ def test_triton_basic_call():
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["packed", "dense"])
|
||||
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
kernel = Mock(return_value=torch.empty(0))
|
||||
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
|
||||
|
||||
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
|
||||
state = torch.empty(2, nheads, dim, dstate)
|
||||
x_shape: tuple[int, ...]
|
||||
B_shape: tuple[int, ...]
|
||||
expected_x_shape: tuple[int, ...]
|
||||
expected_B_shape: tuple[int, ...]
|
||||
if layout == "packed":
|
||||
x_shape = (6, nheads, dim)
|
||||
B_shape = (6, ngroups, dstate)
|
||||
expected_x_shape = (1, 6, nheads, dim)
|
||||
expected_B_shape = (1, 6, ngroups, dstate)
|
||||
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
kernel_max_seqlen = max_seqlen
|
||||
else:
|
||||
x_shape = (batch, max_seqlen, nheads, dim)
|
||||
B_shape = (batch, max_seqlen, ngroups, dstate)
|
||||
expected_x_shape = x_shape
|
||||
expected_B_shape = B_shape
|
||||
cu_seqlens = None
|
||||
kernel_max_seqlen = None
|
||||
x = torch.empty(x_shape)
|
||||
dt = torch.empty_like(x)
|
||||
A = torch.empty(nheads, dim, dstate)
|
||||
B = torch.empty(B_shape)
|
||||
C = torch.empty_like(B)
|
||||
out = torch.empty_like(x)
|
||||
x_cache = torch.empty(2, nheads, 20, dim)
|
||||
dt_cache = torch.empty(2, nheads, 20)
|
||||
B_cache = torch.empty(2, ngroups, 20, dstate)
|
||||
ring_start = torch.zeros(2, dtype=torch.int32)
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32)
|
||||
selective_state_update_replayssm_flashinfer(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
out,
|
||||
x_cache,
|
||||
B_cache,
|
||||
dt_cache,
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
logical_window=16,
|
||||
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=kernel_max_seqlen,
|
||||
update_trackers=False,
|
||||
)
|
||||
|
||||
args = kernel.call_args.args
|
||||
assert args[6].shape == expected_x_shape
|
||||
assert args[7].shape == expected_x_shape
|
||||
assert args[9].shape == expected_B_shape
|
||||
assert args[10].shape == expected_B_shape
|
||||
assert args[11].shape == expected_x_shape
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_ring_len"),
|
||||
("query_start_loc", "expected_shape", "expected_max_seqlen"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 17),
|
||||
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
|
||||
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
def test_replayssm_mixer_selects_mtp_layout(
|
||||
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
|
||||
):
|
||||
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
|
||||
|
||||
mixer = MambaMixer2.__new__(MambaMixer2)
|
||||
torch.nn.Module.__init__(mixer)
|
||||
mixer.prefix = "mixer"
|
||||
mixer.tped_intermediate_size = 0
|
||||
mixer.tped_conv_size = 1
|
||||
mixer.tped_dt_size = 2
|
||||
mixer.num_heads = 2
|
||||
mixer.head_dim = 4
|
||||
mixer.n_groups = mixer.tp_size = 1
|
||||
mixer.ssm_state_size = 8
|
||||
mixer.num_spec = 3
|
||||
mixer.use_replayssm = True
|
||||
mixer.replayssm_buffer_len = 16
|
||||
mixer._commits_replayssm_trackers = True
|
||||
mixer._updates_replayssm_trackers = False
|
||||
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
|
||||
mixer.conv_weights = torch.empty(0)
|
||||
mixer.conv1d = SimpleNamespace(bias=None)
|
||||
mixer.activation = "silu"
|
||||
mixer.A = torch.empty(2)
|
||||
mixer.dt_bias = torch.empty(2)
|
||||
mixer.D = torch.empty(2)
|
||||
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
|
||||
mixer.kv_cache = (
|
||||
torch.empty(3, 1),
|
||||
torch.empty(3, 2, 4, 8),
|
||||
torch.empty(3, 2, 20, 4),
|
||||
torch.empty(3, 2, 20),
|
||||
torch.empty(3, 1, 20, 8),
|
||||
)
|
||||
|
||||
num_decode_tokens = query_start_loc[-1]
|
||||
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
|
||||
metadata = Mamba2AttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=2,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
def split_hidden_states_B_C(values):
|
||||
tokens = values.size(0)
|
||||
return (
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
)
|
||||
|
||||
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
|
||||
kernel = Mock()
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
|
||||
)
|
||||
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
|
||||
monkeypatch.setattr(
|
||||
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
|
||||
)
|
||||
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
|
||||
|
||||
mixer.conv_ssm_forward(
|
||||
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
|
||||
)
|
||||
|
||||
assert kernel.call_args.args[1].shape == expected_shape
|
||||
if expected_max_seqlen is None:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is None
|
||||
else:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "num_speculative_tokens", "expected_ring_len"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 0, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 0, 17),
|
||||
(MambaBackendEnum.FLASHINFER, 3, 20),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(
|
||||
backend, num_speculative_tokens, expected_ring_len
|
||||
):
|
||||
base_shapes = ((64, 3), (8, 4, 16))
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
@@ -247,6 +634,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=backend,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
assert shapes[2:] == (
|
||||
|
||||
@@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
with patch.object(
|
||||
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
max_num_reqs, decode_kwargs = result
|
||||
assert max_num_reqs == expected_num_reqs
|
||||
expected_kwargs = {
|
||||
**PREFILL_KWARGS,
|
||||
"num_tokens": expected_num_reqs * query_len,
|
||||
@@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
force_attention=True,
|
||||
profile_seq_lens=query_len + 1,
|
||||
)
|
||||
assert result == (expected_num_reqs, expected_kwargs)
|
||||
assert decode_kwargs == expected_kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
(dict(use_replayssm=False), True),
|
||||
(dict(backend=MambaBackendEnum.TRITON), True),
|
||||
({}, False),
|
||||
(dict(num_blocks=1), True),
|
||||
],
|
||||
ids=[
|
||||
"replayssm_disabled",
|
||||
"non_flashinfer_backend",
|
||||
"kernel_unavailable",
|
||||
"zero_non_padding_slots",
|
||||
],
|
||||
ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"],
|
||||
)
|
||||
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
with patch.object(
|
||||
@@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
"flashinfer_replayssm_autotune_supported",
|
||||
return_value=flashinfer_supported,
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs))
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers():
|
||||
)
|
||||
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
|
||||
tracked = (
|
||||
*mixer.kv_cache,
|
||||
mixer._replayssm_ring_start,
|
||||
mixer._replayssm_prev_num_accepted,
|
||||
mixer._replayssm_prev_query_len,
|
||||
)
|
||||
|
||||
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)
|
||||
|
||||
@@ -153,6 +153,7 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
use_replayssm=True,
|
||||
use_kda_recoverssm=False,
|
||||
mamba_cache_mode="none",
|
||||
replayssm_buffer_len=16,
|
||||
),
|
||||
num_speculative_tokens=3,
|
||||
model_config=SimpleNamespace(
|
||||
@@ -184,8 +185,9 @@ def test_kda_recoverssm_derivation_is_revalidated():
|
||||
config.cache_config.mamba_cache_mode = "none"
|
||||
|
||||
config.model_config.architecture = "NemotronHForCausalLM"
|
||||
with pytest.raises(ValueError, match="only supported for Kimi-K3 KDA"):
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
config.mamba_config.backend = MambaBackendEnum.FLASHINFER
|
||||
VllmConfig.validate_mamba_cached_kernel(config)
|
||||
assert not config.cache_config.use_kda_recoverssm
|
||||
|
||||
config.model_config.architecture = "KimiLinearForCausalLM"
|
||||
config.parallel_config.pipeline_parallel_size = 2
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.abstract import MambaBase
|
||||
from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import (
|
||||
MiniMaxText01LinearAttention,
|
||||
)
|
||||
@@ -19,6 +21,32 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("use_replayssm", "expected_blocks"), [(False, 3), (True, 0)])
|
||||
def test_replayssm_does_not_reserve_speculative_state_blocks(
|
||||
use_replayssm, expected_blocks
|
||||
):
|
||||
layer = SimpleNamespace(
|
||||
get_state_shape=lambda: ((2,),),
|
||||
get_state_dtype=lambda: (torch.float32,),
|
||||
mamba_type=MambaAttentionBackendEnum.MAMBA2,
|
||||
is_kv_cache_tp_replicated=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
mamba_block_size=1,
|
||||
mamba_page_size_padded=None,
|
||||
mamba_cache_mode="none",
|
||||
use_replayssm=use_replayssm,
|
||||
),
|
||||
num_speculative_tokens=3,
|
||||
)
|
||||
|
||||
spec = MambaBase.get_kv_cache_spec(layer, vllm_config)
|
||||
|
||||
assert spec is not None
|
||||
assert spec.num_speculative_blocks == expected_blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_class, init_kwargs, expected_backend, expected_mamba_type",
|
||||
[
|
||||
|
||||
@@ -16,6 +16,8 @@ from tests.v1.attention.utils import (
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
@@ -191,9 +193,12 @@ REPLAYSSM_BUILD_CASES = {
|
||||
def _make_mamba_spec(
|
||||
buffer_len: int,
|
||||
mamba_backend: MambaBackendEnum,
|
||||
num_speculative_tokens: int = 0,
|
||||
) -> MambaSpec:
|
||||
ring_buffer_len = buffer_len + (
|
||||
1 if mamba_backend == MambaBackendEnum.FLASHINFER else 0
|
||||
1 + num_speculative_tokens
|
||||
if mamba_backend == MambaBackendEnum.FLASHINFER
|
||||
else 0
|
||||
)
|
||||
shapes = (
|
||||
(1, 1),
|
||||
@@ -214,6 +219,7 @@ def _create_replayssm_builder(
|
||||
mamba_cache_mode: str = "none",
|
||||
*,
|
||||
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON,
|
||||
num_speculative_tokens: int = 0,
|
||||
) -> MockMambaBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE
|
||||
@@ -224,21 +230,32 @@ def _create_replayssm_builder(
|
||||
vllm_config.cache_config.replayssm_buffer_len = buffer_len
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.mamba_config.backend = mamba_backend
|
||||
if num_speculative_tokens > 0:
|
||||
# Triton ReplaySSM requires exact synchronous CPU metadata.
|
||||
vllm_config.scheduler_config.async_scheduling = False
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
return MockMambaBuilder(
|
||||
_make_mamba_spec(buffer_len, mamba_backend),
|
||||
_make_mamba_spec(buffer_len, mamba_backend, num_speculative_tokens),
|
||||
["layer0"],
|
||||
vllm_config,
|
||||
DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _build(builder: MockMambaBuilder, case: ReplaySSMBuildCase):
|
||||
def _build(
|
||||
builder: MockMambaBuilder,
|
||||
case: ReplaySSMBuildCase,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
):
|
||||
batch = BatchSpec(seq_lens=case.seq_lens, query_lens=case.query_lens)
|
||||
common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace(
|
||||
is_prefilling=torch.tensor(case.is_prefilling, dtype=torch.bool),
|
||||
replayssm_decode_base_cpu=torch.tensor(case.decode_base, dtype=torch.int32),
|
||||
)
|
||||
return builder.build(0, common)
|
||||
return builder.build(0, common, num_accepted_tokens=num_accepted_tokens)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -270,21 +287,60 @@ def test_resumed_request_differs_from_fresh():
|
||||
assert meta.is_flush_d.tolist()[:2] == [0, 0]
|
||||
|
||||
|
||||
def test_flashinfer_replayssm_scratch_metadata_fresh_decode():
|
||||
checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
|
||||
if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"):
|
||||
pytest.skip("FlashInfer does not expose ReplaySSM scratch allocation")
|
||||
def test_spec_decode_single_token_chunk_synthesizes_acceptance_metadata():
|
||||
builder = _create_replayssm_builder(16, num_speculative_tokens=3)
|
||||
case = REPLAYSSM_BUILD_CASES["leftover_prompt_one_token_flush"]
|
||||
|
||||
builder = _create_replayssm_builder(16, mamba_backend=MambaBackendEnum.FLASHINFER)
|
||||
case = REPLAYSSM_BUILD_CASES["fresh_decode"]
|
||||
meta = _build(builder, case)
|
||||
|
||||
assert meta.write_pos_d is None
|
||||
assert meta.is_flush_d is None
|
||||
assert meta.bc_pre_scratch is None
|
||||
assert meta.replayssm_scratch is not None
|
||||
assert [tensor.shape for tensor in meta.replayssm_scratch] == [
|
||||
(1, 1, 32, 8),
|
||||
(1, 1, 16),
|
||||
(1, 1, 32, 8),
|
||||
]
|
||||
assert meta.query_start_loc_d is not None
|
||||
assert meta.query_start_loc_d.tolist() == [0, 1]
|
||||
assert meta.num_accepted_tokens is not None
|
||||
assert meta.num_accepted_tokens.tolist() == [1]
|
||||
|
||||
|
||||
def test_flashinfer_replayssm_state_indices_are_stable_for_full_cudagraph():
|
||||
checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
|
||||
if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"):
|
||||
pytest.skip("requires FlashInfer ReplaySSM autotuning support")
|
||||
|
||||
builder = _create_replayssm_builder(
|
||||
16,
|
||||
mamba_backend=MambaBackendEnum.FLASHINFER,
|
||||
num_speculative_tokens=3,
|
||||
)
|
||||
builder.compilation_config.cudagraph_mode = CUDAGraphMode.FULL
|
||||
|
||||
first = _build(
|
||||
builder,
|
||||
ReplaySSMBuildCase(
|
||||
seq_lens=[106, 106],
|
||||
query_lens=[1, 1],
|
||||
is_prefilling=[False, False],
|
||||
decode_base=[100, 100],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[],
|
||||
expected_is_flush=[],
|
||||
),
|
||||
)
|
||||
first_indices = first.replayssm_state_indices_d
|
||||
assert first_indices is not None
|
||||
assert first_indices.is_contiguous()
|
||||
first_ptr = first_indices.data_ptr()
|
||||
|
||||
second = _build(
|
||||
builder,
|
||||
ReplaySSMBuildCase(
|
||||
seq_lens=[122, 122],
|
||||
query_lens=[1, 1],
|
||||
is_prefilling=[False, False],
|
||||
decode_base=[116, 116],
|
||||
buffer_len=16,
|
||||
expected_write_pos=[],
|
||||
expected_is_flush=[],
|
||||
),
|
||||
)
|
||||
second_indices = second.replayssm_state_indices_d
|
||||
assert second_indices is not None
|
||||
assert second_indices.data_ptr() == first_ptr
|
||||
assert torch.equal(second_indices, second.state_indices_tensor_d[:, 0])
|
||||
|
||||
@@ -10,11 +10,16 @@ from vllm.v1.metrics.reader import Counter
|
||||
from ...models.utils import check_logprobs_close
|
||||
from ...utils import large_gpu_mark, multi_gpu_test
|
||||
|
||||
# Mamba2 (Nemotron-3) hybrid.
|
||||
# Mamba2 (Nemotron-3) hybrids.
|
||||
MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16"
|
||||
# FlashInfer 0.7.0's standard SSU cannot compile Nano's DIM=80 configuration.
|
||||
MAMBA2_FLASHINFER_MODEL = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4"
|
||||
MODELS = [
|
||||
pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)),
|
||||
]
|
||||
FLASHINFER_MODELS = [
|
||||
pytest.param(MAMBA2_FLASHINFER_MODEL, marks=large_gpu_mark(min_gb=40)),
|
||||
]
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
@@ -22,6 +27,16 @@ PROMPTS = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _use_v1_model_runner_by_default(monkeypatch):
|
||||
# Triton ReplaySSM is V1-only. FlashInfer V2 tests override this locally.
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0")
|
||||
envs.disable_envs_cache()
|
||||
yield
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
def _check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
@@ -29,17 +44,11 @@ def _check_replayssm_parity(
|
||||
tensor_parallel_size=1,
|
||||
mamba_backend: str = "triton",
|
||||
name_1: str = "replayssm",
|
||||
require_v2: bool = False,
|
||||
monkeypatch: pytest.MonkeyPatch | None = None,
|
||||
expected_v2: bool | None = None,
|
||||
):
|
||||
# Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a
|
||||
# near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are
|
||||
# common-mode and only ReplaySSM varies.
|
||||
if require_v2:
|
||||
assert monkeypatch is not None
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
envs.disable_envs_cache()
|
||||
|
||||
# near-tie. Baseline and ReplaySSM use the same backend so only ReplaySSM
|
||||
# behavior varies.
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
@@ -49,14 +58,17 @@ def _check_replayssm_parity(
|
||||
mamba_backend=mamba_backend,
|
||||
)
|
||||
with vllm_runner(model_name, **common) as llm:
|
||||
if require_v2:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
if expected_v2 is not None:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
|
||||
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
with vllm_runner(
|
||||
model_name, use_replayssm=True, replayssm_buffer_len=16, **common
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
**common,
|
||||
) as llm:
|
||||
if require_v2:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
if expected_v2 is not None:
|
||||
assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2
|
||||
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
|
||||
check_logprobs_close(
|
||||
@@ -80,19 +92,101 @@ def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name):
|
||||
_check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_replayssm_flashinfer_decode_matches_baseline_v2(
|
||||
vllm_runner, model_name, monkeypatch
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_decode_matches_baseline(
|
||||
vllm_runner, model_name, monkeypatch, use_v2_model_runner
|
||||
):
|
||||
pytest.importorskip("flashinfer.mamba.checkpointing_ssu")
|
||||
_check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
mamba_backend="flashinfer",
|
||||
name_1="replayssm_flashinfer_v2",
|
||||
require_v2=True,
|
||||
monkeypatch=monkeypatch,
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
_check_replayssm_parity(
|
||||
vllm_runner,
|
||||
model_name,
|
||||
mamba_backend="flashinfer",
|
||||
name_1="replayssm_flashinfer",
|
||||
expected_v2=use_v2_model_runner,
|
||||
)
|
||||
finally:
|
||||
# The context restores the environment before the final cache reset.
|
||||
envs.disable_envs_cache()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"num_speculative_tokens": 3,
|
||||
"prompt_lookup_max": 3,
|
||||
},
|
||||
)
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
outputs_1_lst=replay,
|
||||
name_0="baseline_spec",
|
||||
name_1="replayssm_flashinfer_spec",
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@large_gpu_mark(min_gb=40)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
mamba_backend="flashinfer",
|
||||
tensor_parallel_size=2,
|
||||
disable_log_stats=False,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
|
||||
)
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
with vllm_runner(
|
||||
MAMBA2_FLASHINFER_MODEL,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
**common,
|
||||
) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
outputs = llm.generate_greedy(PROMPTS, max_tokens=32)
|
||||
draft_count = sum(
|
||||
metric.value
|
||||
for metric in llm.llm.get_metrics()
|
||||
if isinstance(metric, Counter)
|
||||
and metric.name == "vllm:spec_decode_num_drafts"
|
||||
)
|
||||
finally:
|
||||
envs.disable_envs_cache()
|
||||
|
||||
# At least one request must run past the 16-token replay window; another
|
||||
# may legitimately stop early on EOS.
|
||||
assert any(len(token_ids) > 16 for token_ids, _ in outputs)
|
||||
assert draft_count > 0
|
||||
|
||||
|
||||
# Prefix spans several mamba blocks; prefix caching only reuses full blocks.
|
||||
|
||||
@@ -1260,6 +1260,8 @@ class _TestReplaySSMMixer(MambaMixer2):
|
||||
self.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
|
||||
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
|
||||
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
|
||||
self._commits_replayssm_trackers = True
|
||||
self._updates_replayssm_trackers = True
|
||||
|
||||
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
|
||||
@@ -1294,19 +1296,19 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only):
|
||||
else:
|
||||
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
|
||||
|
||||
assert (
|
||||
mixers[0]._replayssm_ring_start.data_ptr()
|
||||
== mixers[2]._replayssm_ring_start.data_ptr()
|
||||
tracker_names = (
|
||||
"_replayssm_ring_start",
|
||||
"_replayssm_prev_num_accepted",
|
||||
"_replayssm_prev_query_len",
|
||||
)
|
||||
assert (
|
||||
mixers[0]._replayssm_prev_num_accepted.data_ptr()
|
||||
== mixers[2]._replayssm_prev_num_accepted.data_ptr()
|
||||
)
|
||||
assert (
|
||||
mixers[1]._replayssm_ring_start.data_ptr()
|
||||
!= mixers[0]._replayssm_ring_start.data_ptr()
|
||||
)
|
||||
# Group {0, 2} shares trackers; layer 2 (not 0) updates after both run.
|
||||
for tracker_name in tracker_names:
|
||||
group_tracker = getattr(mixers[0], tracker_name)
|
||||
assert group_tracker.data_ptr() == getattr(mixers[2], tracker_name).data_ptr()
|
||||
assert group_tracker.data_ptr() != getattr(mixers[1], tracker_name).data_ptr()
|
||||
assert group_tracker.shape == (4,)
|
||||
assert torch.count_nonzero(group_tracker) == 0
|
||||
|
||||
assert [m._commits_replayssm_trackers for m in mixers] == [True, True, False]
|
||||
assert [m._updates_replayssm_trackers for m in mixers] == [False, True, True]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user