From 7dbd0a8d22d20eb938994ad7a9158e02f30c9ce6 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 24 Sep 2026 18:29:53 -0700 Subject: [PATCH] [Bugfix][MRV2] Treat padded prompt tails as spec-decode rows for hybrid models (#58434) Signed-off-by: Nick Hill Co-authored-by: Claude Opus 5.5 (1M context) --- .../worker/test_mamba_hybrid_model_state.py | 80 +++++++++++++++++++ .../worker/gpu/model_states/mamba_hybrid.py | 9 ++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_mamba_hybrid_model_state.py b/tests/v1/worker/test_mamba_hybrid_model_state.py index c65679059a..19f77c13e6 100644 --- a/tests/v1/worker/test_mamba_hybrid_model_state.py +++ b/tests/v1/worker/test_mamba_hybrid_model_state.py @@ -4,9 +4,16 @@ from types import SimpleNamespace from unittest.mock import Mock +import numpy as np import pytest import torch +from tests.v1.attention.test_gdn_metadata_builder import ( + BLOCK_SIZE, + DEVICE, + _create_gdn_builder, +) +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata from vllm.config.compilation import CUDAGraphMode from vllm.platforms import current_platform from vllm.v1.attention.backends.recoverssm_metadata import ( @@ -59,6 +66,79 @@ def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> Non assert build_attn_metadata.call_args.kwargs["positions"] is positions +def test_padded_prompt_tail_builds_as_spec_decode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A one-token prompt tail over prior state, padded with K placeholder + drafts (e.g. a P/D decode-node arrival), must reach the GDN builder as a + spec-decode row. Built as a prefill, the placeholder tokens are folded into + the recurrent state and can't be rolled back. + """ + k = 3 + state = object.__new__(MambaHybridModelState) + state.vllm_config = SimpleNamespace(num_speculative_tokens=k) + state.model_config = SimpleNamespace(max_model_len=8192) + state._align_mode = False + state.recoverssm = None + state.num_accepted_tokens_gpu = torch.ones(4, dtype=torch.int32) + + # A verify decode, a padded prompt tail (128 of 129 prompt tokens already + # computed), and a fresh prompt chunk of the same length. + query_lens = [k + 1] * 3 + seq_lens = [50, 128 + k + 1, k + 1] + is_prefilling = [False, True, True] + query_start_loc = np.array([0, 4, 8, 12], dtype=np.int32) + input_batch = SimpleNamespace( + num_reqs=3, + num_tokens=12, + num_reqs_after_padding=3, + num_tokens_after_padding=12, + idx_mapping=torch.arange(3), + query_start_loc_np=query_start_loc, + query_start_loc=torch.from_numpy(query_start_loc), + num_scheduled_tokens=np.array(query_lens, dtype=np.int32), + num_draft_tokens_per_req=np.array([k, k, 0], dtype=np.int32), + max_query_len=None, + seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32), + is_prefilling_np=np.array(is_prefilling), + prefill_len_np=np.array([40, 129, 100], dtype=np.int32), + num_computed_prefill_tokens_np=np.array([40, 128, 0], dtype=np.int32), + dcp_local_seq_lens=None, + positions=torch.zeros(12, dtype=torch.int64), + prompt_lens=None, + ) + build_attn_metadata = Mock(return_value={}) + monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata) + state.prepare_attn( + input_batch=input_batch, + cudagraph_mode=CUDAGraphMode.NONE, + block_tables=(), + slot_mappings=torch.empty(0, dtype=torch.int64), + attn_groups=[], + kv_cache_config=Mock(), + ) + mamba_metadata = build_attn_metadata.call_args.kwargs[ + "model_specific_attn_metadata" + ] + + builder = _create_gdn_builder(num_speculative_tokens=k) + common = create_common_attn_metadata( + BatchSpec(seq_lens=seq_lens, query_lens=query_lens), BLOCK_SIZE, DEVICE + ).replace(is_prefilling=torch.tensor(is_prefilling)) + meta = builder.build( + common_prefix_len=0, + common_attn_metadata=common, + num_accepted_tokens=mamba_metadata.num_accepted_tokens, + num_decode_draft_tokens_cpu=mamba_metadata.num_decode_draft_tokens_cpu, + ) + + # Only the fresh prompt chunk needs the prefill kernels. + assert meta.num_spec_decodes == 2 + assert meta.num_prefills == 1 + assert meta.num_prefill_tokens == k + 1 + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") @pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)]) def test_postprocess_state_scalar_with_int32_mapping( diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index df527bd891..2b73b55bac 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -281,7 +281,14 @@ class MambaHybridModelState(DefaultModelState): # Test request state, not num_scheduled_tokens == draft_count+1: # adaptive rewrites num_scheduled_tokens to an even split, so that # equality rarely holds and would demote every verify row to decode. - is_decode = (~input_batch.is_prefilling_np) & ( + # A one-token prompt tail over prior state that the scheduler padded + # with placeholder drafts is also a spec-decode row: the prefill + # kernels can't roll the placeholders back. + num_computed = input_batch.num_computed_prefill_tokens_np + is_prompt_tail = (num_computed > 0) & ( + input_batch.prefill_len_np - num_computed == 1 + ) + is_decode = (~input_batch.is_prefilling_np | is_prompt_tail) & ( input_batch.num_scheduled_tokens > 0 ) spec_decode_mask = (num_draft_tokens_per_req > 0) & is_decode