[Bugfix] Preserve Mamba state for padded prompt tails (#55178)
Signed-off-by: Nathan Sala <natouda@gmail.com> Signed-off-by: Nathan Sala <33957625+natsala13@users.noreply.github.com>
This commit is contained in:
@@ -141,6 +141,7 @@ class Mamba2AttentionMetadataBuilder(
|
||||
common_attn_metadata,
|
||||
num_accepted_tokens=kwargs.get("num_accepted_tokens"),
|
||||
prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"),
|
||||
num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"),
|
||||
)
|
||||
|
||||
seq_idx_p = None
|
||||
|
||||
@@ -273,6 +273,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
*,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
prev_last_scheduled_idx: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
**kwargs: Any,
|
||||
) -> M:
|
||||
"""
|
||||
@@ -283,6 +284,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
common_attn_metadata,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
prev_last_scheduled_idx=prev_last_scheduled_idx,
|
||||
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||
)
|
||||
|
||||
def _compute_chunk_metadata(
|
||||
@@ -459,6 +461,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
*,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
prev_last_scheduled_idx: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
) -> M:
|
||||
"""
|
||||
Compute metadata common to both Mamba1 and Mamba2.
|
||||
@@ -484,10 +487,28 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound
|
||||
assert seq_lens_cpu is not None
|
||||
query_lens_cpu = torch.diff(common_attn_metadata.query_start_loc_cpu)
|
||||
single_token_prefill_rows = is_prefilling & (query_lens_cpu == 1)
|
||||
# First-token prefills have no prior Mamba state and must stay prefills.
|
||||
has_prior_state = seq_lens_cpu > 1
|
||||
prefill_to_decode = single_token_prefill_rows & has_prior_state
|
||||
|
||||
# First prompt chunks have no prior Mamba state and must stay prefills.
|
||||
has_prior_state = seq_lens_cpu > query_lens_cpu
|
||||
stateful_prefill_rows = is_prefilling & has_prior_state
|
||||
|
||||
# One-token prefills with prior state can use the decode/update path.
|
||||
prefill_to_decode = stateful_prefill_rows & (query_lens_cpu == 1)
|
||||
|
||||
# The scheduler may pad a one-token remote prompt tail with placeholder
|
||||
# drafts to retain the uniform K+1 decode graph. This is a speculative
|
||||
# decode transaction even though the real token is still in the prompt:
|
||||
# the decode kernels keep h(N) in the running slot and h(N+i) in scratch
|
||||
# slots, so normal acceptance rollback remains valid. The prefill kernels
|
||||
# only return h(N+K) and cannot roll the placeholders back.
|
||||
if num_decode_draft_tokens_cpu is not None:
|
||||
padded_prompt_tail_rows = (
|
||||
stateful_prefill_rows
|
||||
& (num_decode_draft_tokens_cpu >= 0)
|
||||
& (query_lens_cpu == num_decode_draft_tokens_cpu + 1)
|
||||
)
|
||||
prefill_to_decode |= padded_prompt_tail_rows
|
||||
|
||||
if torch.any(prefill_to_decode).item():
|
||||
# ReplaySSM handles these rows as single-token flushes (see the
|
||||
# write-position derivation below), same as the baseline decode path.
|
||||
|
||||
Reference in New Issue
Block a user