From 874df9373dab532543a0229fb2f144f7c14093ae Mon Sep 17 00:00:00 2001 From: Nathan Sala <33957625+natsala13@users.noreply.github.com> Date: Sat, 5 Sep 2026 02:44:28 +0300 Subject: [PATCH] [Bugfix] Preserve Mamba state for padded prompt tails (#55178) Signed-off-by: Nathan Sala Signed-off-by: Nathan Sala <33957625+natsala13@users.noreply.github.com> --- vllm/v1/attention/backends/mamba2_attn.py | 1 + vllm/v1/attention/backends/mamba_attn.py | 29 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index f44de4131c..189ef54b23 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -141,6 +141,7 @@ class Mamba2AttentionMetadataBuilder( common_attn_metadata, num_accepted_tokens=kwargs.get("num_accepted_tokens"), prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"), + num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"), ) seq_idx_p = None diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index b1a0c2d0ef..1b9ee263db 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -273,6 +273,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): *, num_accepted_tokens: torch.Tensor | None = None, prev_last_scheduled_idx: torch.Tensor | None = None, + num_decode_draft_tokens_cpu: torch.Tensor | None = None, **kwargs: Any, ) -> M: """ @@ -283,6 +284,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): common_attn_metadata, num_accepted_tokens=num_accepted_tokens, prev_last_scheduled_idx=prev_last_scheduled_idx, + num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) def _compute_chunk_metadata( @@ -459,6 +461,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): *, num_accepted_tokens: torch.Tensor | None = None, prev_last_scheduled_idx: torch.Tensor | None = None, + num_decode_draft_tokens_cpu: torch.Tensor | None = None, ) -> M: """ Compute metadata common to both Mamba1 and Mamba2. @@ -484,10 +487,28 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound assert seq_lens_cpu is not None query_lens_cpu = torch.diff(common_attn_metadata.query_start_loc_cpu) - single_token_prefill_rows = is_prefilling & (query_lens_cpu == 1) - # First-token prefills have no prior Mamba state and must stay prefills. - has_prior_state = seq_lens_cpu > 1 - prefill_to_decode = single_token_prefill_rows & has_prior_state + + # First prompt chunks have no prior Mamba state and must stay prefills. + has_prior_state = seq_lens_cpu > query_lens_cpu + stateful_prefill_rows = is_prefilling & has_prior_state + + # One-token prefills with prior state can use the decode/update path. + prefill_to_decode = stateful_prefill_rows & (query_lens_cpu == 1) + + # The scheduler may pad a one-token remote prompt tail with placeholder + # drafts to retain the uniform K+1 decode graph. This is a speculative + # decode transaction even though the real token is still in the prompt: + # the decode kernels keep h(N) in the running slot and h(N+i) in scratch + # slots, so normal acceptance rollback remains valid. The prefill kernels + # only return h(N+K) and cannot roll the placeholders back. + if num_decode_draft_tokens_cpu is not None: + padded_prompt_tail_rows = ( + stateful_prefill_rows + & (num_decode_draft_tokens_cpu >= 0) + & (query_lens_cpu == num_decode_draft_tokens_cpu + 1) + ) + prefill_to_decode |= padded_prompt_tail_rows + if torch.any(prefill_to_decode).item(): # ReplaySSM handles these rows as single-token flushes (see the # write-position derivation below), same as the baseline decode path.