[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
7e583e615c
commit
deb148e730
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
FlashInferSSUBackend,
|
||||
TritonSSUBackend,
|
||||
commit_replayssm_ring_trackers,
|
||||
get_mamba_ssu_backend,
|
||||
initialize_mamba_ssu_backend,
|
||||
reset_replayssm_ring_trackers,
|
||||
selective_state_update,
|
||||
selective_state_update_replayssm_flashinfer,
|
||||
update_replayssm_ring_trackers,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -32,6 +38,13 @@ try:
|
||||
except ImportError:
|
||||
HAS_FLASHINFER = False
|
||||
|
||||
try:
|
||||
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
|
||||
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
|
||||
except ImportError:
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_backend_state():
|
||||
@@ -47,6 +60,7 @@ def restore_backend_state():
|
||||
def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
update_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
logical_window=16,
|
||||
ring_buffer_len=17,
|
||||
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("accepted_sequence", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[4] * 22,
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 8, 4),
|
||||
(0, 12, 4),
|
||||
(0, 16, 4),
|
||||
(16, 4, 4),
|
||||
(16, 8, 4),
|
||||
(16, 12, 4),
|
||||
(16, 16, 4),
|
||||
(12, 4, 4),
|
||||
(12, 8, 4),
|
||||
(12, 12, 4),
|
||||
(12, 16, 4),
|
||||
(8, 4, 4),
|
||||
(8, 8, 4),
|
||||
(8, 12, 4),
|
||||
(8, 16, 4),
|
||||
(4, 4, 4),
|
||||
(4, 8, 4),
|
||||
(4, 12, 4),
|
||||
(4, 16, 4),
|
||||
(0, 4, 4),
|
||||
],
|
||||
id="all-accepted",
|
||||
),
|
||||
pytest.param(
|
||||
[4, 4, 0, 3, 4, 2, 4, 1],
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 4, 4),
|
||||
(0, 7, 4),
|
||||
(0, 11, 4),
|
||||
(0, 13, 4),
|
||||
(13, 4, 4),
|
||||
(13, 5, 4),
|
||||
],
|
||||
id="mixed",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
|
||||
logical_window = 16
|
||||
num_speculative_tokens = 3
|
||||
query_len = 1 + num_speculative_tokens
|
||||
ring_buffer_len = logical_window + 1 + num_speculative_tokens
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in accepted_sequence:
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window,
|
||||
ring_buffer_len,
|
||||
)
|
||||
snapshot = (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
)
|
||||
observed.append(snapshot)
|
||||
assert snapshot[1] + snapshot[2] <= ring_buffer_len
|
||||
|
||||
assert observed == expected
|
||||
|
||||
|
||||
def test_replayssm_resume_resets_commit_history():
|
||||
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([3], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 4)
|
||||
|
||||
|
||||
def test_replayssm_commit_tracker_ragged_query_lengths():
|
||||
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in ([4, 2], [3, 1]):
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
observed.append(
|
||||
[
|
||||
(
|
||||
ring_start[slot].item(),
|
||||
prev_num_accepted[slot].item(),
|
||||
prev_query_len[slot].item(),
|
||||
)
|
||||
for slot in (1, 2)
|
||||
]
|
||||
)
|
||||
|
||||
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["commit", "reset"])
|
||||
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
|
||||
num_states = 3
|
||||
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor(
|
||||
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
if operation == "commit":
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
expected_valid = (2, 5, 4)
|
||||
else:
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
expected_valid = (0, 0, 0)
|
||||
|
||||
assert (
|
||||
ring_start.tolist(),
|
||||
prev_num_accepted.tolist(),
|
||||
prev_query_len.tolist(),
|
||||
) == (
|
||||
[11, expected_valid[0], 33],
|
||||
[11, expected_valid[1], 33],
|
||||
[11, expected_valid[2], 33],
|
||||
)
|
||||
|
||||
|
||||
def _kv_cache_config_with_ssu(
|
||||
@@ -231,14 +442,190 @@ def test_triton_basic_call():
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["packed", "dense"])
|
||||
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
kernel = Mock(return_value=torch.empty(0))
|
||||
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
|
||||
|
||||
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
|
||||
state = torch.empty(2, nheads, dim, dstate)
|
||||
x_shape: tuple[int, ...]
|
||||
B_shape: tuple[int, ...]
|
||||
expected_x_shape: tuple[int, ...]
|
||||
expected_B_shape: tuple[int, ...]
|
||||
if layout == "packed":
|
||||
x_shape = (6, nheads, dim)
|
||||
B_shape = (6, ngroups, dstate)
|
||||
expected_x_shape = (1, 6, nheads, dim)
|
||||
expected_B_shape = (1, 6, ngroups, dstate)
|
||||
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
kernel_max_seqlen = max_seqlen
|
||||
else:
|
||||
x_shape = (batch, max_seqlen, nheads, dim)
|
||||
B_shape = (batch, max_seqlen, ngroups, dstate)
|
||||
expected_x_shape = x_shape
|
||||
expected_B_shape = B_shape
|
||||
cu_seqlens = None
|
||||
kernel_max_seqlen = None
|
||||
x = torch.empty(x_shape)
|
||||
dt = torch.empty_like(x)
|
||||
A = torch.empty(nheads, dim, dstate)
|
||||
B = torch.empty(B_shape)
|
||||
C = torch.empty_like(B)
|
||||
out = torch.empty_like(x)
|
||||
x_cache = torch.empty(2, nheads, 20, dim)
|
||||
dt_cache = torch.empty(2, nheads, 20)
|
||||
B_cache = torch.empty(2, ngroups, 20, dstate)
|
||||
ring_start = torch.zeros(2, dtype=torch.int32)
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32)
|
||||
selective_state_update_replayssm_flashinfer(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
out,
|
||||
x_cache,
|
||||
B_cache,
|
||||
dt_cache,
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
logical_window=16,
|
||||
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=kernel_max_seqlen,
|
||||
update_trackers=False,
|
||||
)
|
||||
|
||||
args = kernel.call_args.args
|
||||
assert args[6].shape == expected_x_shape
|
||||
assert args[7].shape == expected_x_shape
|
||||
assert args[9].shape == expected_B_shape
|
||||
assert args[10].shape == expected_B_shape
|
||||
assert args[11].shape == expected_x_shape
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_ring_len"),
|
||||
("query_start_loc", "expected_shape", "expected_max_seqlen"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 17),
|
||||
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
|
||||
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
def test_replayssm_mixer_selects_mtp_layout(
|
||||
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
|
||||
):
|
||||
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
|
||||
|
||||
mixer = MambaMixer2.__new__(MambaMixer2)
|
||||
torch.nn.Module.__init__(mixer)
|
||||
mixer.prefix = "mixer"
|
||||
mixer.tped_intermediate_size = 0
|
||||
mixer.tped_conv_size = 1
|
||||
mixer.tped_dt_size = 2
|
||||
mixer.num_heads = 2
|
||||
mixer.head_dim = 4
|
||||
mixer.n_groups = mixer.tp_size = 1
|
||||
mixer.ssm_state_size = 8
|
||||
mixer.num_spec = 3
|
||||
mixer.use_replayssm = True
|
||||
mixer.replayssm_buffer_len = 16
|
||||
mixer._commits_replayssm_trackers = True
|
||||
mixer._updates_replayssm_trackers = False
|
||||
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
|
||||
mixer.conv_weights = torch.empty(0)
|
||||
mixer.conv1d = SimpleNamespace(bias=None)
|
||||
mixer.activation = "silu"
|
||||
mixer.A = torch.empty(2)
|
||||
mixer.dt_bias = torch.empty(2)
|
||||
mixer.D = torch.empty(2)
|
||||
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
|
||||
mixer.kv_cache = (
|
||||
torch.empty(3, 1),
|
||||
torch.empty(3, 2, 4, 8),
|
||||
torch.empty(3, 2, 20, 4),
|
||||
torch.empty(3, 2, 20),
|
||||
torch.empty(3, 1, 20, 8),
|
||||
)
|
||||
|
||||
num_decode_tokens = query_start_loc[-1]
|
||||
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
|
||||
metadata = Mamba2AttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=2,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
num_computed_tokens_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
def split_hidden_states_B_C(values):
|
||||
tokens = values.size(0)
|
||||
return (
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
)
|
||||
|
||||
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
|
||||
kernel = Mock()
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
|
||||
)
|
||||
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
|
||||
monkeypatch.setattr(
|
||||
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
|
||||
)
|
||||
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
|
||||
|
||||
mixer.conv_ssm_forward(
|
||||
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
|
||||
)
|
||||
|
||||
assert kernel.call_args.args[1].shape == expected_shape
|
||||
if expected_max_seqlen is None:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is None
|
||||
else:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "num_speculative_tokens", "expected_ring_len"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 0, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 0, 17),
|
||||
(MambaBackendEnum.FLASHINFER, 3, 20),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(
|
||||
backend, num_speculative_tokens, expected_ring_len
|
||||
):
|
||||
base_shapes = ((64, 3), (8, 4, 16))
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
@@ -247,6 +634,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=backend,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
assert shapes[2:] == (
|
||||
|
||||
Reference in New Issue
Block a user