[Bugfix] Preserve Mamba state for padded prompt tails (#55178)

Signed-off-by: Nathan Sala <natouda@gmail.com>
Signed-off-by: Nathan Sala <33957625+natsala13@users.noreply.github.com>
This commit is contained in:
Nathan Sala
2026-09-04 23:44:28 +00:00
committed by GitHub
parent eb74fbb3e7
commit 874df9373d
2 changed files with 26 additions and 4 deletions
@@ -141,6 +141,7 @@ class Mamba2AttentionMetadataBuilder(
common_attn_metadata,
num_accepted_tokens=kwargs.get("num_accepted_tokens"),
prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"),
num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"),
)
seq_idx_p = None
+25 -4
View File
@@ -273,6 +273,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
*,
num_accepted_tokens: torch.Tensor | None = None,
prev_last_scheduled_idx: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
**kwargs: Any,
) -> M:
"""
@@ -283,6 +284,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
common_attn_metadata,
num_accepted_tokens=num_accepted_tokens,
prev_last_scheduled_idx=prev_last_scheduled_idx,
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
)
def _compute_chunk_metadata(
@@ -459,6 +461,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
*,
num_accepted_tokens: torch.Tensor | None = None,
prev_last_scheduled_idx: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
) -> M:
"""
Compute metadata common to both Mamba1 and Mamba2.
@@ -484,10 +487,28 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
assert seq_lens_cpu is not None
query_lens_cpu = torch.diff(common_attn_metadata.query_start_loc_cpu)
single_token_prefill_rows = is_prefilling & (query_lens_cpu == 1)
# First-token prefills have no prior Mamba state and must stay prefills.
has_prior_state = seq_lens_cpu > 1
prefill_to_decode = single_token_prefill_rows & has_prior_state
# First prompt chunks have no prior Mamba state and must stay prefills.
has_prior_state = seq_lens_cpu > query_lens_cpu
stateful_prefill_rows = is_prefilling & has_prior_state
# One-token prefills with prior state can use the decode/update path.
prefill_to_decode = stateful_prefill_rows & (query_lens_cpu == 1)
# The scheduler may pad a one-token remote prompt tail with placeholder
# drafts to retain the uniform K+1 decode graph. This is a speculative
# decode transaction even though the real token is still in the prompt:
# the decode kernels keep h(N) in the running slot and h(N+i) in scratch
# slots, so normal acceptance rollback remains valid. The prefill kernels
# only return h(N+K) and cannot roll the placeholders back.
if num_decode_draft_tokens_cpu is not None:
padded_prompt_tail_rows = (
stateful_prefill_rows
& (num_decode_draft_tokens_cpu >= 0)
& (query_lens_cpu == num_decode_draft_tokens_cpu + 1)
)
prefill_to_decode |= padded_prompt_tail_rows
if torch.any(prefill_to_decode).item():
# ReplaySSM handles these rows as single-token flushes (see the
# write-position derivation below), same as the baseline decode path.