[Bugfix][Spec Decode] Reserve the bonus KV slot for fill-in DSpark (#59105)

Signed-off-by: Evgeny Savinov <notime.sea@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
Evgeny Savinov
2026-10-05 17:35:51 +00:00
committed by GitHub
co-authored by Codex Tomas Ruiz
parent 182247257f
commit 877ddcc456
2 changed files with 31 additions and 19 deletions
+24 -14
View File
@@ -314,24 +314,28 @@ def test_reserved_block_count_matches_real_kv_cache_manager():
@pytest.mark.parametrize(
("method", "expected"),
("method", "draft_hf_config", "expected"),
[
("eagle", NUM_SPEC_STEPS),
("eagle3", NUM_SPEC_STEPS),
("mtp", NUM_SPEC_STEPS),
("dspark", NUM_SPEC_STEPS),
("draft_model", NUM_SPEC_STEPS),
("eagle", None, NUM_SPEC_STEPS),
("eagle3", None, NUM_SPEC_STEPS),
("mtp", None, NUM_SPEC_STEPS),
("dspark", None, NUM_SPEC_STEPS),
("dspark", {"sample_from_anchor": True}, NUM_SPEC_STEPS),
("dspark", {"sample_from_anchor": False}, NUM_SPEC_STEPS + 1),
("draft_model", None, NUM_SPEC_STEPS),
# DFlash's in-fill decoding adds a query for the last sampled token.
("dflash", NUM_SPEC_STEPS + 1),
("ngram", 0),
("ngram_gpu", 0),
("medusa", 0),
("mlp_speculator", 0),
("suffix", 0),
("extract_hidden_states", 0),
("dflash", None, NUM_SPEC_STEPS + 1),
("ngram", None, 0),
("ngram_gpu", None, 0),
("medusa", None, 0),
("mlp_speculator", None, 0),
("suffix", None, 0),
("extract_hidden_states", None, 0),
],
)
def test_num_lookahead_tokens_per_method(method: str, expected: int):
def test_num_lookahead_tokens_per_method(
method: str, draft_hf_config: dict | None, expected: int
):
"""`VllmConfig.num_lookahead_tokens` is the single source of the reservation.
Both the scheduler and the warmup read it, so a wrong answer here silently
@@ -350,6 +354,12 @@ def test_num_lookahead_tokens_per_method(method: str, expected: int):
speculative_config = object.__new__(SpeculativeConfig)
object.__setattr__(speculative_config, "method", method)
object.__setattr__(speculative_config, "num_speculative_tokens", NUM_SPEC_STEPS)
hf_config = SimpleNamespace(**(draft_hf_config or {}))
object.__setattr__(
speculative_config,
"draft_model_config",
SimpleNamespace(hf_config=hf_config),
)
config = _Config()
config.speculative_config = speculative_config
+7 -5
View File
@@ -640,13 +640,15 @@ class VllmConfig:
speculative_config = self.speculative_config
if speculative_config is None:
return 0
if speculative_config.use_dflash():
# DFlash requires an extra lookahead slot since it uses in-fill-style
# decoding instead of standard next-token sampling, so it has a query
# for the last sampled token plus queries for each draft token.
dspark_fill_in = speculative_config.use_dspark() and not getattr(
speculative_config.draft_model_config.hf_config, "sample_from_anchor", True
)
if speculative_config.use_dflash() or dspark_fill_in:
# Fill-in drafting uses a bonus query plus one query per draft token.
# DSpark's anchor-sampling layout does not need the extra slot.
return self.num_speculative_tokens + 1
if speculative_config.use_eagle() or speculative_config.uses_draft_model():
# DSpark (covered by use_eagle) drafts a block of num_speculative_tokens
# Anchor-sampling DSpark drafts a block of num_speculative_tokens
# query tokens in which the anchor itself is the first prediction
# position (no separate bonus query), so it needs exactly
# num_speculative_tokens lookahead slots.