From c7a305ec10f3ec06c2784f1258162483c5fc8d73 Mon Sep 17 00:00:00 2001 From: sungsoo ha Date: Tue, 13 Oct 2026 03:09:17 -0700 Subject: [PATCH] [Bugfix][Mamba] Store conv_kernel - 1 inputs in Mamba2 prefill checkpoints (#60797) Signed-off-by: Sungsoo Ha Co-authored-by: Claude Opus 5.5 Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> --- .../mamba/test_mamba2_checkpoint_state.py | 16 +++++++++++----- vllm/model_executor/layers/mamba/mamba_mixer2.py | 1 + .../layers/mamba/ops/ssd_checkpoint.py | 7 +++++-- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/kernels/mamba/test_mamba2_checkpoint_state.py b/tests/kernels/mamba/test_mamba2_checkpoint_state.py index 54fcf80205..a06d894792 100644 --- a/tests/kernels/mamba/test_mamba2_checkpoint_state.py +++ b/tests/kernels/mamba/test_mamba2_checkpoint_state.py @@ -114,23 +114,27 @@ def test_checkpoint_does_not_perturb_the_final_state(): ) +@pytest.mark.parametrize("spec_slots", [0, 3]) @pytest.mark.parametrize("dim_first", [True, False]) -def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first): +def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots): """The store reads the checkpoint row out of every chunk state. Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the program's own row, so the store only passes if ``chunk_idx`` is used. Row 1 declines and must write nothing. The conv input is a column slice of a wider projection, the SD conv layout is a transposed view, and SSM rows - are padded pages, matching the strides the mixer passes. + are padded pages, matching the strides the mixer passes. Spec decoding + widens the conv state, and the store must still fill only the first + ``state_len`` slots with the newest inputs. """ dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8 projection = torch.randn(64, dim + 8, device=DEVICE) conv_input = projection[:, :dim] + num_slots = state_len + spec_slots if dim_first: - conv_state = torch.zeros(3, dim, state_len, device=DEVICE) + conv_state = torch.zeros(3, dim, num_slots, device=DEVICE) else: - conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2) + conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2) varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE) # Blocks are pages holding more than the SSM state, so rows are padded. row = heads * head_dim * d_state @@ -152,12 +156,14 @@ def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first): varlen_states, ssm_state, torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE), + state_len=state_len, ) exact = dict(rtol=0, atol=0) torch.testing.assert_close( - conv_state[2], conv_input[32 - state_len : 32].T, **exact + conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact ) + assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots" torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact) assert not conv_state[:2].any() and not ssm_storage[:2].any() assert not ssm_storage[:, row:].any(), "wrote into the page padding" diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index 14c483122a..e310aaf40f 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -890,6 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer): varlen_states, ssm_state, query_start_loc_p, + state_len=self.conv_kernel_size - 1, ) ssm_state[state_indices_tensor_p] = final_states if ring_start is not None and self._updates_replayssm_trackers: diff --git a/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py b/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py index 25bfcedbdc..fdd8ac8f0e 100644 --- a/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py +++ b/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py @@ -69,6 +69,8 @@ def store_prefill_checkpoint( chunk_states: torch.Tensor, ssm_state: torch.Tensor, query_start_loc: torch.Tensor, + *, + state_len: int, ) -> None: """Write each prefill row's mid-prefill checkpoint into its paged block. @@ -78,14 +80,15 @@ def store_prefill_checkpoint( checkpoint: Per-row offsets and destination blocks. chunk_idx: Per-row index into ``chunk_states`` of the checkpoint. x: Pre-conv activations, ``(dim, num_prefill_tokens)``. - conv_state: Paged conv state, ``(num_blocks, dim, state_len)``. + conv_state: Paged conv state, ``(num_blocks, dim, state_len + num_spec)``. chunk_states: SSD states at every chunk end. ssm_state: Paged SSM state. query_start_loc: Prefill query start locations. + state_len: ``conv_kernel_size - 1``, not the spec-widened conv width. + Readers take the first ``state_len`` slots as the newest inputs. """ dim = x.size(0) - state_len = conv_state.size(-1) ssm_row_size = chunk_states[0].numel() block_size = 1024 grid = (