[Bugfix] Route every speculation-capable row through the speculative path so recurrent state stays consistent (#56531)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor Agent
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4a705c7b54
commit
0eac152707
@@ -0,0 +1,249 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""`GPUModelRunner._prepare_inputs` must tell a chunked-prefill continuation
|
||||
row apart from a genuine zero-draft decode row.
|
||||
|
||||
Both kinds of row are absent from `scheduled_spec_decode_tokens` and both
|
||||
schedule exactly one token, so `num_decode_draft_tokens` starts at the
|
||||
placeholder `-1` for either. The `zero_draft_decode_mask` in
|
||||
`gpu_model_runner.py` is what tells them apart afterwards, using
|
||||
`num_computed_tokens >= num_prompt_tokens`: a prefill continuation must keep
|
||||
`-1` (it carries no drafts and no accepted-token offset to apply), while a
|
||||
zero-draft decode row must become `0` (it still has to run the speculative
|
||||
path so the recurrent backends apply that offset). Getting this wrong in
|
||||
either direction was the root cause behind issue #40875: Kimi-Linear crashed
|
||||
with an illegal memory access when prefill tails were misclassified as
|
||||
zero-draft decodes.
|
||||
|
||||
Each batch below also includes one row with a real draft token, so
|
||||
`scheduled_spec_decode_tokens` is non-empty even under the pre-fix
|
||||
`use_spec_decode = len(scheduled_spec_decode_tokens) > 0` predicate. That
|
||||
keeps the assertions targeted at the mask fix alone, not the unrelated
|
||||
`use_spec_decode` predicate fix (`self.speculative_config is not None`) that
|
||||
landed in the same patch.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
ModelConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
SpeculativeConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.core.sched.output import CachedRequestData, NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
NUM_BLOCKS = 10
|
||||
DEVICE_TYPE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
def _get_vllm_config() -> VllmConfig:
|
||||
model_config = ModelConfig(model="facebook/opt-125m", dtype="float16", seed=42)
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=10,
|
||||
max_num_batched_tokens=512,
|
||||
max_model_len=512,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
)
|
||||
cache_config = CacheConfig(
|
||||
block_size=BLOCK_SIZE, gpu_memory_utilization=0.9, cache_dtype="auto"
|
||||
)
|
||||
cache_config.kv_cache_layout = "LBNHC"
|
||||
return VllmConfig(
|
||||
model_config=model_config,
|
||||
cache_config=cache_config,
|
||||
scheduler_config=scheduler_config,
|
||||
parallel_config=ParallelConfig(),
|
||||
# ngram needs no draft model weights, so the fixture stays as cheap as
|
||||
# the speculative-decoding-free one in test_gpu_model_runner.py.
|
||||
speculative_config=SpeculativeConfig(method="ngram", num_speculative_tokens=3),
|
||||
)
|
||||
|
||||
|
||||
def _initialize_kv_cache(runner: GPUModelRunner) -> None:
|
||||
attn_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=runner.model_config.get_num_kv_heads(runner.parallel_config),
|
||||
head_size=runner.model_config.get_head_size(),
|
||||
dtype=runner.kv_cache_dtype,
|
||||
)
|
||||
tensor_size = attn_spec.page_size_bytes * NUM_BLOCKS
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=NUM_BLOCKS,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=tensor_size,
|
||||
layers=["layer.0"],
|
||||
layer_stride=tensor_size,
|
||||
block_stride=attn_spec.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(layer_names=["layer.0"], kv_cache_spec=attn_spec)
|
||||
],
|
||||
)
|
||||
runner.kv_cache_config = kv_cache_config
|
||||
runner.input_batch = InputBatch(
|
||||
max_num_reqs=runner.max_num_reqs,
|
||||
max_model_len=runner.max_model_len,
|
||||
max_num_batched_tokens=runner.max_num_tokens,
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def spec_decode_model_runner(dist_init):
|
||||
vllm_config = _get_vllm_config()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
model_config = vllm_config.model_config
|
||||
num_heads = model_config.get_num_kv_heads(vllm_config.parallel_config)
|
||||
head_size = model_config.get_head_size()
|
||||
vllm_config.compilation_config.static_forward_context["layer.0"] = Attention(
|
||||
num_heads, head_size, 0.1
|
||||
)
|
||||
runner = GPUModelRunner(vllm_config, DEVICE_TYPE)
|
||||
_initialize_kv_cache(runner)
|
||||
yield runner
|
||||
|
||||
|
||||
def _new_request(
|
||||
req_id: str, prompt_len: int, num_computed_tokens: int
|
||||
) -> tuple[NewRequestData, int]:
|
||||
"""A request with `prompt_len` prompt tokens, `num_computed_tokens` of
|
||||
which are already computed (e.g. a prefix-cache hit), scheduled for the
|
||||
single remaining/next token this step."""
|
||||
return (
|
||||
NewRequestData(
|
||||
req_id=req_id,
|
||||
prompt_token_ids=list(range(prompt_len)),
|
||||
mm_features=[],
|
||||
sampling_params=SamplingParams(),
|
||||
pooling_params=None,
|
||||
block_ids=([0],),
|
||||
num_computed_tokens=num_computed_tokens,
|
||||
lora_request=None,
|
||||
),
|
||||
1,
|
||||
)
|
||||
|
||||
|
||||
def test_prefill_continuation_keeps_placeholder_decode_zero_draft_becomes_zero(
|
||||
spec_decode_model_runner,
|
||||
):
|
||||
"""One request has one token left to prefill (`num_computed_tokens ==
|
||||
num_prompt_tokens - 1`); another has already finished its prompt and is
|
||||
now decoding with no draft tokens proposed this step; a third is decoding
|
||||
with a real draft. `decode_with_draft` gives the batch a non-empty
|
||||
`scheduled_spec_decode_tokens`, so this also exercises the `-1`-vs-`0`
|
||||
distinction on the pre-fix definition of `use_spec_decode` (`len(sched
|
||||
uled_spec_decode_tokens) > 0`), isolating the mask fix from the unrelated
|
||||
`use_spec_decode` predicate fix that landed in the same patch."""
|
||||
runner = spec_decode_model_runner
|
||||
|
||||
prefill_req, prefill_sched = _new_request(
|
||||
"prefill_tail", prompt_len=11, num_computed_tokens=10
|
||||
)
|
||||
decode_req, decode_sched = _new_request(
|
||||
"decode_zero_draft", prompt_len=3, num_computed_tokens=3
|
||||
)
|
||||
drafted_req, _ = _new_request(
|
||||
"decode_with_draft", prompt_len=3, num_computed_tokens=3
|
||||
)
|
||||
num_scheduled_tokens = {
|
||||
"prefill_tail": prefill_sched,
|
||||
"decode_zero_draft": decode_sched,
|
||||
"decode_with_draft": 2, # 1 draft token + 1 bonus token
|
||||
}
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[prefill_req, decode_req, drafted_req],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
total_num_scheduled_tokens=sum(num_scheduled_tokens.values()),
|
||||
scheduled_spec_decode_tokens={"decode_with_draft": [99]},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
runner._update_states(scheduler_output)
|
||||
runner._prepare_inputs(
|
||||
scheduler_output,
|
||||
np.array(
|
||||
[num_scheduled_tokens[r] for r in runner.input_batch.req_ids],
|
||||
dtype=np.int32,
|
||||
),
|
||||
)
|
||||
|
||||
prefill_idx = runner.input_batch.req_id_to_index["prefill_tail"]
|
||||
decode_idx = runner.input_batch.req_id_to_index["decode_zero_draft"]
|
||||
drafted_idx = runner.input_batch.req_id_to_index["decode_with_draft"]
|
||||
assert runner.num_decode_draft_tokens.np[prefill_idx] == -1
|
||||
assert runner.num_decode_draft_tokens.np[decode_idx] == 0
|
||||
assert runner.num_decode_draft_tokens.np[drafted_idx] == 1
|
||||
|
||||
|
||||
def test_prefill_tail_of_two_or_three_tokens_also_keeps_the_placeholder(
|
||||
spec_decode_model_runner,
|
||||
):
|
||||
"""A chunked-prefill tail can be 1, 2, or 3 tokens (`long_prefill_token_
|
||||
threshold` rounding); the mask only fires for `num_scheduled_tokens == 1`,
|
||||
so wider tails must keep `-1` regardless of how much prompt is left. A
|
||||
third, drafted decode row keeps `scheduled_spec_decode_tokens` non-empty
|
||||
so the assertions hold on the pre-fix `use_spec_decode` predicate too."""
|
||||
runner = spec_decode_model_runner
|
||||
|
||||
tail_2_req, _ = _new_request("tail_2", prompt_len=12, num_computed_tokens=10)
|
||||
tail_3_req, _ = _new_request("tail_3", prompt_len=13, num_computed_tokens=10)
|
||||
drafted_req, _ = _new_request(
|
||||
"decode_with_draft", prompt_len=3, num_computed_tokens=3
|
||||
)
|
||||
num_scheduled_tokens = {"tail_2": 2, "tail_3": 3, "decode_with_draft": 2}
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=[tail_2_req, tail_3_req, drafted_req],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
total_num_scheduled_tokens=sum(num_scheduled_tokens.values()),
|
||||
scheduled_spec_decode_tokens={"decode_with_draft": [99]},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
runner._update_states(scheduler_output)
|
||||
runner._prepare_inputs(
|
||||
scheduler_output,
|
||||
np.array(
|
||||
[num_scheduled_tokens[r] for r in runner.input_batch.req_ids],
|
||||
dtype=np.int32,
|
||||
),
|
||||
)
|
||||
|
||||
tail_2_idx = runner.input_batch.req_id_to_index["tail_2"]
|
||||
tail_3_idx = runner.input_batch.req_id_to_index["tail_3"]
|
||||
assert runner.num_decode_draft_tokens.np[tail_2_idx] == -1
|
||||
assert runner.num_decode_draft_tokens.np[tail_3_idx] == -1
|
||||
@@ -140,6 +140,177 @@ def test_padded_prompt_tail_builds_as_spec_decode(
|
||||
assert meta.num_prefill_tokens == k + 1
|
||||
|
||||
|
||||
def _mamba_hybrid_state(num_speculative_tokens: int) -> MambaHybridModelState:
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.vllm_config = SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
state.model_config = SimpleNamespace(max_model_len=8192)
|
||||
state._align_mode = False
|
||||
state.recoverssm = None
|
||||
# A request that accepted 3 drafts (plus the bonus token) on the previous
|
||||
# step; the others have never had a draft accepted.
|
||||
state.num_accepted_tokens_gpu = torch.tensor([4, 1, 1], dtype=torch.int32)
|
||||
return state
|
||||
|
||||
|
||||
def _input_batch(
|
||||
num_scheduled_tokens: list[int],
|
||||
num_draft_tokens_per_req: list[int] | None,
|
||||
is_prefilling: list[bool],
|
||||
num_reqs_after_padding: int | None = None,
|
||||
) -> SimpleNamespace:
|
||||
"""`num_draft_tokens_per_req=None` is a batch where nobody proposed a
|
||||
draft this step, e.g. because ngram found no match for anyone."""
|
||||
num_reqs = len(num_scheduled_tokens)
|
||||
query_start_loc = np.zeros(num_reqs + 1, dtype=np.int32)
|
||||
query_start_loc[1:] = np.cumsum(num_scheduled_tokens)
|
||||
num_tokens = int(query_start_loc[-1])
|
||||
seq_lens = torch.full((num_reqs,), 10, dtype=torch.int32)
|
||||
return SimpleNamespace(
|
||||
num_reqs=num_reqs,
|
||||
num_reqs_after_padding=num_reqs_after_padding or num_reqs,
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_after_padding=num_tokens,
|
||||
query_start_loc_np=query_start_loc,
|
||||
num_scheduled_tokens=np.array(num_scheduled_tokens, dtype=np.int32),
|
||||
max_query_len=None,
|
||||
seq_lens_cpu_upper_bound=seq_lens,
|
||||
is_prefilling_np=np.array(is_prefilling),
|
||||
prefill_runs_as_decode_np=None,
|
||||
num_draft_tokens_per_req=None
|
||||
if num_draft_tokens_per_req is None
|
||||
else np.array(num_draft_tokens_per_req, dtype=np.int32),
|
||||
idx_mapping=torch.arange(num_reqs),
|
||||
query_start_loc=torch.from_numpy(query_start_loc),
|
||||
seq_lens=seq_lens,
|
||||
dcp_local_seq_lens=None,
|
||||
positions=torch.zeros(num_tokens, dtype=torch.int64),
|
||||
prompt_lens=None,
|
||||
)
|
||||
|
||||
|
||||
def _prepare_attn_metadata(
|
||||
state: MambaHybridModelState,
|
||||
monkeypatch,
|
||||
input_batch: SimpleNamespace | None = None,
|
||||
cudagraph_mode: CUDAGraphMode = CUDAGraphMode.NONE,
|
||||
) -> SimpleNamespace:
|
||||
captured = {}
|
||||
|
||||
def fake_build_attn_metadata(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return Mock()
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.model_states.mamba_hybrid.build_attn_metadata",
|
||||
fake_build_attn_metadata,
|
||||
)
|
||||
|
||||
if input_batch is None:
|
||||
input_batch = _input_batch([1, 1], None, [False, False])
|
||||
state.prepare_attn(
|
||||
input_batch=input_batch,
|
||||
cudagraph_mode=cudagraph_mode,
|
||||
block_tables=(),
|
||||
slot_mappings={},
|
||||
attn_groups=[],
|
||||
kv_cache_config=Mock(),
|
||||
)
|
||||
return captured["model_specific_attn_metadata"]
|
||||
|
||||
|
||||
def test_prepare_attn_keeps_the_accepted_offset_when_the_batch_drafted_nothing(
|
||||
monkeypatch,
|
||||
):
|
||||
"""`num_accepted_tokens` must reach the attention builders whenever the
|
||||
speculative config is on, even when *this* batch scheduled no draft
|
||||
tokens at all. Before the fix, `use_spec_decode` was gated on this step's
|
||||
scheduled drafts (`len(scheduled_spec_decode_tokens) > 0` in the V1
|
||||
runner) rather than on the speculative config, so a request that accepted
|
||||
drafts on a previous step would silently lose that offset the moment
|
||||
ngram found no match for anyone in the batch.
|
||||
"""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=3)
|
||||
|
||||
metadata = _prepare_attn_metadata(state, monkeypatch)
|
||||
|
||||
assert metadata.num_accepted_tokens is not None
|
||||
assert metadata.num_accepted_tokens.tolist() == [4, 1]
|
||||
assert metadata.num_decode_draft_tokens_cpu is not None
|
||||
|
||||
|
||||
def test_prepare_attn_omits_the_offset_without_a_speculative_config(monkeypatch):
|
||||
"""Mirrors the check above: with speculative decoding off entirely, there
|
||||
is no accepted-token offset to carry, and the metadata must say so
|
||||
explicitly (`None`) rather than a stale or default value."""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=0)
|
||||
|
||||
metadata = _prepare_attn_metadata(state, monkeypatch)
|
||||
|
||||
assert metadata.num_accepted_tokens is None
|
||||
assert metadata.num_decode_draft_tokens_cpu is None
|
||||
|
||||
|
||||
def test_zero_draft_decode_row_is_marked_as_a_speculative_row(monkeypatch):
|
||||
"""A decode row that got no drafts still has to reach the speculative path.
|
||||
|
||||
The recurrent backends only apply a row's accepted-token offset on that
|
||||
path, and a row that accepted drafts on the previous step keeps that offset
|
||||
even when the current step proposes nothing (the scheduler drops drafts
|
||||
whenever the token budget truncates the step). Marking such a row with the
|
||||
-1 sentinel sends it to the plain decode path, which reads the state at
|
||||
column 0 and ignores the offset.
|
||||
"""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=3)
|
||||
input_batch = _input_batch([3, 1, 1], [2, 0, 0], [False, False, True])
|
||||
|
||||
metadata = _prepare_attn_metadata(state, monkeypatch, input_batch)
|
||||
|
||||
assert metadata.num_decode_draft_tokens_cpu.tolist() == [
|
||||
2, # decode row with drafts
|
||||
0, # decode row with no drafts: speculative path, offset applied
|
||||
-1, # one-token prefill tail: no offset to apply, stays a plain row
|
||||
]
|
||||
|
||||
|
||||
def test_batch_without_any_drafts_still_marks_its_decode_rows(monkeypatch):
|
||||
"""`num_draft_tokens_per_req` is None whenever no row in the batch drafted
|
||||
anything. Those rows are ordinary decode rows carrying an offset from the
|
||||
previous step, so the absence of drafts in this step must not be read as
|
||||
speculative decoding being off."""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=3)
|
||||
input_batch = _input_batch([1, 1], None, [False, True])
|
||||
|
||||
metadata = _prepare_attn_metadata(state, monkeypatch, input_batch)
|
||||
|
||||
assert metadata.num_decode_draft_tokens_cpu.tolist() == [0, -1]
|
||||
|
||||
|
||||
def test_padded_rows_keep_the_sentinel(monkeypatch):
|
||||
"""Full CUDA-graph capture pads the batch; padded rows describe no request
|
||||
and must not be picked up as speculative rows."""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=3)
|
||||
input_batch = _input_batch([1, 1], [0, 0], [False, False], num_reqs_after_padding=4)
|
||||
|
||||
metadata = _prepare_attn_metadata(
|
||||
state, monkeypatch, input_batch, cudagraph_mode=CUDAGraphMode.FULL
|
||||
)
|
||||
|
||||
assert metadata.num_decode_draft_tokens_cpu.tolist() == [0, 0, -1, -1]
|
||||
|
||||
|
||||
def test_chunked_prefill_tail_of_two_or_three_tokens_keeps_the_sentinel(
|
||||
monkeypatch,
|
||||
):
|
||||
"""A prefill continuation is excluded by `is_prefilling`, not by its token
|
||||
count, so tails wider than one token are covered by the same rule."""
|
||||
state = _mamba_hybrid_state(num_speculative_tokens=3)
|
||||
input_batch = _input_batch([2, 3], [0, 0], [True, True])
|
||||
|
||||
metadata = _prepare_attn_metadata(state, monkeypatch, input_batch)
|
||||
|
||||
assert metadata.num_decode_draft_tokens_cpu.tolist() == [-1, -1]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
|
||||
def test_postprocess_state_scalar_with_int32_mapping(
|
||||
|
||||
Reference in New Issue
Block a user