[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)

This commit is contained in:
Siddharth Bedekar
2026-05-13 21:04:03 -04:00
committed by GitHub
parent 665f9c4253
commit f51f6844f9
7 changed files with 264 additions and 13 deletions
+18
View File
@@ -1286,6 +1286,24 @@ def test_eagle_draft_model_config():
assert draft_model_config.architecture == "EagleLlamaForCausalLM"
def test_draft_sample_method_probabilistic_is_accepted():
speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=1,
draft_sample_method="probabilistic",
)
assert speculative_config.draft_sample_method == "probabilistic"
def test_draft_sample_method_gumbel_is_rejected():
with pytest.raises(ValidationError):
SpeculativeConfig(
method="ngram",
num_speculative_tokens=1,
draft_sample_method="gumbel",
)
def test_ir_op_priority_default():
"""Test that IR op priority defaults are set correctly."""
from vllm.config.kernel import IrOpPriorityConfig
+101
View File
@@ -50,6 +50,8 @@ def _create_proposer(
num_speculative_tokens: int,
attention_backend: str | None = None,
parallel_drafting: bool = False,
rejection_sample_method: str = "standard",
draft_sample_method: str = "greedy",
) -> EagleProposer:
# Method-dependent setup
if method == "eagle":
@@ -81,6 +83,8 @@ def _create_proposer(
method=method,
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=parallel_drafting,
rejection_sample_method=rejection_sample_method,
draft_sample_method=draft_sample_method,
)
if parallel_drafting:
# Overwrite pard_token to avoid crash during init
@@ -997,6 +1001,103 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
assert torch.equal(result, expected_tokens)
def test_propose_stores_probabilistic_draft_probs(monkeypatch):
device = torch.device(DEVICE_TYPE)
batch_size = 2
seq_lens = [5, 3]
total_tokens = sum(seq_lens)
num_speculative_tokens = 3
vocab_size = 8
proposer = _create_proposer(
"draft_model",
num_speculative_tokens,
rejection_sample_method="standard",
draft_sample_method="probabilistic",
)
hidden_size = proposer.hidden_size
expanded_total_tokens = total_tokens + batch_size
model_mock = mock.MagicMock()
forward_returns = []
logits_returns = []
for step in range(num_speculative_tokens):
token_count = expanded_total_tokens if step == 0 else batch_size
forward_returns.append(torch.zeros(token_count, hidden_size, device=device))
logits = torch.full((batch_size, vocab_size), -10.0, device=device)
logits[0, step + 1] = 5.0
logits[1, step + 3] = 4.0
logits_returns.append(logits)
model_mock.side_effect = forward_returns
model_mock.compute_logits.side_effect = logits_returns
proposer.model = model_mock
proposer._draft_attn_layer_names = {"layer.0"}
def fake_compute_probs(logits, sampling_metadata):
probs = torch.softmax(logits, dim=-1)
return probs.argmax(dim=-1), probs
monkeypatch.setattr(
"vllm.v1.spec_decode.llm_base_proposer.compute_probs_and_sample_next_token",
fake_compute_probs,
)
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=seq_lens)
common_attn_metadata = create_common_attn_metadata(
batch_spec,
block_size=BLOCK_SIZE,
device=device,
)
attn_metadata_builder_cls, _ = try_get_attention_backend(
AttentionBackendEnum.FLASH_ATTN
)
attn_metadata_builder = attn_metadata_builder_cls(
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
layer_names=proposer._draft_attn_layer_names,
vllm_config=proposer.vllm_config,
device=device,
)
proposer.runner = mock.MagicMock()
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
proposer.draft_attn_groups = [mock_attn_group]
sampling_metadata = mock.MagicMock()
sampling_metadata.all_greedy = False
result = proposer.propose(
target_token_ids=torch.randint(0, vocab_size, (total_tokens,), device=device),
target_positions=torch.cat(
[
torch.arange(seq_lens[0], device=device),
torch.arange(seq_lens[1], device=device),
]
),
target_hidden_states=torch.randn(total_tokens, hidden_size, device=device),
next_token_ids=torch.randint(
0, vocab_size, (batch_size,), dtype=torch.int32, device=device
),
token_indices_to_sample=None,
common_attn_metadata=common_attn_metadata,
sampling_metadata=sampling_metadata,
)
assert result.shape == (batch_size, num_speculative_tokens)
draft_probs = proposer.take_last_draft_probs()
assert draft_probs is not None
assert draft_probs.shape == (batch_size, num_speculative_tokens, vocab_size)
for step, expected_logits in enumerate(logits_returns):
assert torch.allclose(
draft_probs[:, step, :],
torch.softmax(expected_logits, dim=-1),
)
def test_set_inputs_first_pass_dflash():
"""
Test for DFlash set_inputs_first_pass.
+35
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
import pytest
@@ -40,6 +41,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheTensor,
)
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
from vllm.v1.worker.gpu_input_batch import InputBatch
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
from vllm.v1.worker.utils import AttentionGroup, select_common_block_size
@@ -747,6 +749,39 @@ def test_reload_weights_before_load_model(model_runner):
model_runner.reload_weights()
def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
runner = object.__new__(GPUModelRunner)
runner.use_async_scheduling = False
runner.input_batch = SimpleNamespace(
sampling_metadata=Mock(spec=SamplingMetadata),
update_async_output_token_ids=Mock(),
req_ids=["req_a", "req_b", "req_c"],
)
runner.rejection_sampler = Mock(return_value="sampler_output")
runner.sampler = Mock()
runner._draft_prob_req_ids = ["req_c", "req_a", "req_b"]
runner._draft_probs = torch.arange(3 * 3 * 4, dtype=torch.float32).reshape(3, 3, 4)
spec_decode_metadata = SpecDecodeMetadata.make_dummy(
[[1, 2], [], [3]],
device=torch.device("cpu"),
)
logits = torch.randn(6, 4)
output = GPUModelRunner._sample(runner, logits, spec_decode_metadata)
assert output == "sampler_output"
passed_draft_probs = runner.rejection_sampler.call_args.args[1]
expected_draft_probs = torch.cat(
[
runner._draft_probs[1, :2],
runner._draft_probs[0, :1],
],
dim=0,
)
assert torch.equal(passed_draft_probs, expected_draft_probs)
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
torch.set_default_dtype(torch.float16)
layer_0 = "model.layers.0.self_attn.attn"