[Bugfix][Spec Decode] Reserve the bonus KV slot for fill-in DSpark (#59105)
Signed-off-by: Evgeny Savinov <notime.sea@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Tomas Ruiz <tomas.ruiz.te@gmail.com>
This commit is contained in:
co-authored by
Codex
Tomas Ruiz
parent
182247257f
commit
877ddcc456
@@ -314,24 +314,28 @@ def test_reserved_block_count_matches_real_kv_cache_manager():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "expected"),
|
||||
("method", "draft_hf_config", "expected"),
|
||||
[
|
||||
("eagle", NUM_SPEC_STEPS),
|
||||
("eagle3", NUM_SPEC_STEPS),
|
||||
("mtp", NUM_SPEC_STEPS),
|
||||
("dspark", NUM_SPEC_STEPS),
|
||||
("draft_model", NUM_SPEC_STEPS),
|
||||
("eagle", None, NUM_SPEC_STEPS),
|
||||
("eagle3", None, NUM_SPEC_STEPS),
|
||||
("mtp", None, NUM_SPEC_STEPS),
|
||||
("dspark", None, NUM_SPEC_STEPS),
|
||||
("dspark", {"sample_from_anchor": True}, NUM_SPEC_STEPS),
|
||||
("dspark", {"sample_from_anchor": False}, NUM_SPEC_STEPS + 1),
|
||||
("draft_model", None, NUM_SPEC_STEPS),
|
||||
# DFlash's in-fill decoding adds a query for the last sampled token.
|
||||
("dflash", NUM_SPEC_STEPS + 1),
|
||||
("ngram", 0),
|
||||
("ngram_gpu", 0),
|
||||
("medusa", 0),
|
||||
("mlp_speculator", 0),
|
||||
("suffix", 0),
|
||||
("extract_hidden_states", 0),
|
||||
("dflash", None, NUM_SPEC_STEPS + 1),
|
||||
("ngram", None, 0),
|
||||
("ngram_gpu", None, 0),
|
||||
("medusa", None, 0),
|
||||
("mlp_speculator", None, 0),
|
||||
("suffix", None, 0),
|
||||
("extract_hidden_states", None, 0),
|
||||
],
|
||||
)
|
||||
def test_num_lookahead_tokens_per_method(method: str, expected: int):
|
||||
def test_num_lookahead_tokens_per_method(
|
||||
method: str, draft_hf_config: dict | None, expected: int
|
||||
):
|
||||
"""`VllmConfig.num_lookahead_tokens` is the single source of the reservation.
|
||||
|
||||
Both the scheduler and the warmup read it, so a wrong answer here silently
|
||||
@@ -350,6 +354,12 @@ def test_num_lookahead_tokens_per_method(method: str, expected: int):
|
||||
speculative_config = object.__new__(SpeculativeConfig)
|
||||
object.__setattr__(speculative_config, "method", method)
|
||||
object.__setattr__(speculative_config, "num_speculative_tokens", NUM_SPEC_STEPS)
|
||||
hf_config = SimpleNamespace(**(draft_hf_config or {}))
|
||||
object.__setattr__(
|
||||
speculative_config,
|
||||
"draft_model_config",
|
||||
SimpleNamespace(hf_config=hf_config),
|
||||
)
|
||||
|
||||
config = _Config()
|
||||
config.speculative_config = speculative_config
|
||||
|
||||
+7
-5
@@ -640,13 +640,15 @@ class VllmConfig:
|
||||
speculative_config = self.speculative_config
|
||||
if speculative_config is None:
|
||||
return 0
|
||||
if speculative_config.use_dflash():
|
||||
# DFlash requires an extra lookahead slot since it uses in-fill-style
|
||||
# decoding instead of standard next-token sampling, so it has a query
|
||||
# for the last sampled token plus queries for each draft token.
|
||||
dspark_fill_in = speculative_config.use_dspark() and not getattr(
|
||||
speculative_config.draft_model_config.hf_config, "sample_from_anchor", True
|
||||
)
|
||||
if speculative_config.use_dflash() or dspark_fill_in:
|
||||
# Fill-in drafting uses a bonus query plus one query per draft token.
|
||||
# DSpark's anchor-sampling layout does not need the extra slot.
|
||||
return self.num_speculative_tokens + 1
|
||||
if speculative_config.use_eagle() or speculative_config.uses_draft_model():
|
||||
# DSpark (covered by use_eagle) drafts a block of num_speculative_tokens
|
||||
# Anchor-sampling DSpark drafts a block of num_speculative_tokens
|
||||
# query tokens in which the anchor itself is the first prediction
|
||||
# position (no separate bonus query), so it needs exactly
|
||||
# num_speculative_tokens lookahead slots.
|
||||
|
||||
Reference in New Issue
Block a user