[Bugfix][Spec Decode] Wire draft_probs into probabilistic draft_model rejection (#40269)
This commit is contained in:
@@ -1286,6 +1286,24 @@ def test_eagle_draft_model_config():
|
||||
assert draft_model_config.architecture == "EagleLlamaForCausalLM"
|
||||
|
||||
|
||||
def test_draft_sample_method_probabilistic_is_accepted():
|
||||
speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
draft_sample_method="probabilistic",
|
||||
)
|
||||
assert speculative_config.draft_sample_method == "probabilistic"
|
||||
|
||||
|
||||
def test_draft_sample_method_gumbel_is_rejected():
|
||||
with pytest.raises(ValidationError):
|
||||
SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=1,
|
||||
draft_sample_method="gumbel",
|
||||
)
|
||||
|
||||
|
||||
def test_ir_op_priority_default():
|
||||
"""Test that IR op priority defaults are set correctly."""
|
||||
from vllm.config.kernel import IrOpPriorityConfig
|
||||
|
||||
@@ -50,6 +50,8 @@ def _create_proposer(
|
||||
num_speculative_tokens: int,
|
||||
attention_backend: str | None = None,
|
||||
parallel_drafting: bool = False,
|
||||
rejection_sample_method: str = "standard",
|
||||
draft_sample_method: str = "greedy",
|
||||
) -> EagleProposer:
|
||||
# Method-dependent setup
|
||||
if method == "eagle":
|
||||
@@ -81,6 +83,8 @@ def _create_proposer(
|
||||
method=method,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=parallel_drafting,
|
||||
rejection_sample_method=rejection_sample_method,
|
||||
draft_sample_method=draft_sample_method,
|
||||
)
|
||||
if parallel_drafting:
|
||||
# Overwrite pard_token to avoid crash during init
|
||||
@@ -997,6 +1001,103 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
|
||||
assert torch.equal(result, expected_tokens)
|
||||
|
||||
|
||||
def test_propose_stores_probabilistic_draft_probs(monkeypatch):
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
batch_size = 2
|
||||
seq_lens = [5, 3]
|
||||
total_tokens = sum(seq_lens)
|
||||
num_speculative_tokens = 3
|
||||
vocab_size = 8
|
||||
|
||||
proposer = _create_proposer(
|
||||
"draft_model",
|
||||
num_speculative_tokens,
|
||||
rejection_sample_method="standard",
|
||||
draft_sample_method="probabilistic",
|
||||
)
|
||||
hidden_size = proposer.hidden_size
|
||||
expanded_total_tokens = total_tokens + batch_size
|
||||
|
||||
model_mock = mock.MagicMock()
|
||||
forward_returns = []
|
||||
logits_returns = []
|
||||
for step in range(num_speculative_tokens):
|
||||
token_count = expanded_total_tokens if step == 0 else batch_size
|
||||
forward_returns.append(torch.zeros(token_count, hidden_size, device=device))
|
||||
logits = torch.full((batch_size, vocab_size), -10.0, device=device)
|
||||
logits[0, step + 1] = 5.0
|
||||
logits[1, step + 3] = 4.0
|
||||
logits_returns.append(logits)
|
||||
|
||||
model_mock.side_effect = forward_returns
|
||||
model_mock.compute_logits.side_effect = logits_returns
|
||||
proposer.model = model_mock
|
||||
proposer._draft_attn_layer_names = {"layer.0"}
|
||||
|
||||
def fake_compute_probs(logits, sampling_metadata):
|
||||
probs = torch.softmax(logits, dim=-1)
|
||||
return probs.argmax(dim=-1), probs
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.spec_decode.llm_base_proposer.compute_probs_and_sample_next_token",
|
||||
fake_compute_probs,
|
||||
)
|
||||
|
||||
batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=seq_lens)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec,
|
||||
block_size=BLOCK_SIZE,
|
||||
device=device,
|
||||
)
|
||||
|
||||
attn_metadata_builder_cls, _ = try_get_attention_backend(
|
||||
AttentionBackendEnum.FLASH_ATTN
|
||||
)
|
||||
attn_metadata_builder = attn_metadata_builder_cls(
|
||||
kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config),
|
||||
layer_names=proposer._draft_attn_layer_names,
|
||||
vllm_config=proposer.vllm_config,
|
||||
device=device,
|
||||
)
|
||||
proposer.runner = mock.MagicMock()
|
||||
mock_attn_group = mock.MagicMock()
|
||||
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
|
||||
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
|
||||
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
|
||||
proposer.draft_attn_groups = [mock_attn_group]
|
||||
|
||||
sampling_metadata = mock.MagicMock()
|
||||
sampling_metadata.all_greedy = False
|
||||
|
||||
result = proposer.propose(
|
||||
target_token_ids=torch.randint(0, vocab_size, (total_tokens,), device=device),
|
||||
target_positions=torch.cat(
|
||||
[
|
||||
torch.arange(seq_lens[0], device=device),
|
||||
torch.arange(seq_lens[1], device=device),
|
||||
]
|
||||
),
|
||||
target_hidden_states=torch.randn(total_tokens, hidden_size, device=device),
|
||||
next_token_ids=torch.randint(
|
||||
0, vocab_size, (batch_size,), dtype=torch.int32, device=device
|
||||
),
|
||||
token_indices_to_sample=None,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
sampling_metadata=sampling_metadata,
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, num_speculative_tokens)
|
||||
|
||||
draft_probs = proposer.take_last_draft_probs()
|
||||
assert draft_probs is not None
|
||||
assert draft_probs.shape == (batch_size, num_speculative_tokens, vocab_size)
|
||||
for step, expected_logits in enumerate(logits_returns):
|
||||
assert torch.allclose(
|
||||
draft_probs[:, step, :],
|
||||
torch.softmax(expected_logits, dim=-1),
|
||||
)
|
||||
|
||||
|
||||
def test_set_inputs_first_pass_dflash():
|
||||
"""
|
||||
Test for DFlash set_inputs_first_pass.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -40,6 +41,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheTensor,
|
||||
)
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.spec_decode.metadata import SpecDecodeMetadata
|
||||
from vllm.v1.worker.gpu_input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.utils import AttentionGroup, select_common_block_size
|
||||
@@ -747,6 +749,39 @@ def test_reload_weights_before_load_model(model_runner):
|
||||
model_runner.reload_weights()
|
||||
|
||||
|
||||
def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.use_async_scheduling = False
|
||||
runner.input_batch = SimpleNamespace(
|
||||
sampling_metadata=Mock(spec=SamplingMetadata),
|
||||
update_async_output_token_ids=Mock(),
|
||||
req_ids=["req_a", "req_b", "req_c"],
|
||||
)
|
||||
runner.rejection_sampler = Mock(return_value="sampler_output")
|
||||
runner.sampler = Mock()
|
||||
runner._draft_prob_req_ids = ["req_c", "req_a", "req_b"]
|
||||
runner._draft_probs = torch.arange(3 * 3 * 4, dtype=torch.float32).reshape(3, 3, 4)
|
||||
|
||||
spec_decode_metadata = SpecDecodeMetadata.make_dummy(
|
||||
[[1, 2], [], [3]],
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
logits = torch.randn(6, 4)
|
||||
|
||||
output = GPUModelRunner._sample(runner, logits, spec_decode_metadata)
|
||||
|
||||
assert output == "sampler_output"
|
||||
passed_draft_probs = runner.rejection_sampler.call_args.args[1]
|
||||
expected_draft_probs = torch.cat(
|
||||
[
|
||||
runner._draft_probs[1, :2],
|
||||
runner._draft_probs[0, :1],
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
assert torch.equal(passed_draft_probs, expected_draft_probs)
|
||||
|
||||
|
||||
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
|
||||
torch.set_default_dtype(torch.float16)
|
||||
layer_0 = "model.layers.0.self_attn.attn"
|
||||
|
||||
Reference in New Issue
Block a user