[Spec Decode] DFlash2: local convolution + candidate selector (#52816)

This commit is contained in:
Zihan Zhang
2026-08-20 22:27:22 -07:00
committed by GitHub
parent df344bf36f
commit b389ac2946
14 changed files with 866 additions and 44 deletions
+8
View File
@@ -1501,6 +1501,14 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
max_model_len=8192, # Reduce max len to ensure test runs in low-VRAM CI env
max_num_seqs=32,
),
"DFlash2DraftModel": _HfExamplesInfo(
"Qwen/Qwen3.8-27B",
speculative_model="z-lab/Qwen3.8-27B-DFlash2",
is_available_online=False,
use_original_num_layers=True,
max_model_len=8192,
max_num_seqs=32,
),
"DFlashLagunaForCausalLM": _HfExamplesInfo(
"poolside/Laguna-XS-2.1-NVFP4",
speculative_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
+24
View File
@@ -199,6 +199,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
),
)
config._dflash_needs_multi_kv_group = lambda: False
config._is_dflash2_draft = lambda: False
config._is_default_v2_model_runner_model = lambda: (
VllmConfig._is_default_v2_model_runner_model(config)
)
@@ -292,6 +293,29 @@ def test_v2_model_runner_supports_extract_hidden_states():
assert config._get_v2_model_runner_unsupported_features() == []
def test_dflash2_draft_forces_v2_model_runner():
"""A DFlash2 draft must reach the V2 speculator, the only one that runs its
candidate selector; on V1 it would draft as DFlash1 without raising."""
def config(method, architectures):
return SimpleNamespace(
speculative_config=SimpleNamespace(
method=method,
draft_model_config=SimpleNamespace(architectures=architectures),
)
)
assert VllmConfig._is_dflash2_draft(config("dflash", ["DFlash2DraftModel"]))
assert not VllmConfig._is_dflash2_draft(config("dflash", ["DFlashDraftModel"]))
assert not VllmConfig._is_dflash2_draft(config("eagle", ["DFlash2DraftModel"]))
assert not VllmConfig._is_dflash2_draft(SimpleNamespace(speculative_config=None))
assert not VllmConfig._is_dflash2_draft(
SimpleNamespace(
speculative_config=SimpleNamespace(method="dflash", draft_model_config=None)
)
)
@pytest.mark.parametrize(
("use_v2_model_runner", "expected_capture_sizes"),
[
+118
View File
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
@pytest.mark.parametrize("block_size", [5, 8])
def test_grouped_conv_matches_reference(block_size: int):
torch.manual_seed(0)
batch, taps, num_groups, group_size = 3, 3, 4, 2
hidden = torch.randn(batch * block_size, num_groups * group_size)
delta = torch.randn(batch * block_size, taps, num_groups)
base = torch.randn(taps, num_groups * group_size)
actual = _grouped_conv(
hidden, delta, base, block_size, num_groups, group_size, taps
)
hidden_blocks = hidden.view(batch, block_size, num_groups, group_size)
expected = torch.zeros_like(hidden_blocks)
base = base.view(taps, num_groups, group_size)
delta = delta.view(batch, block_size, taps, num_groups)
for position in range(block_size):
for tap in range(min(taps, position + 1)):
expected[:, position] += (
base[tap] + delta[:, position, tap, :, None]
) * hidden_blocks[:, position - tap]
torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2))
def test_selector_edges_match_sequential_reference():
torch.manual_seed(1)
batch, steps, top_k, rank = 2, 4, 3, 5
vocab = 17
predecessors = torch.randn(vocab, rank)
successors = torch.randn(vocab, rank)
candidate_ids = torch.randint(vocab, (batch, steps, top_k))
unary = torch.randn(batch, steps, top_k)
hidden = torch.randn(batch, steps, rank)
anchors = torch.randint(vocab, (batch,))
actual = _score_edges(
predecessors,
successors,
candidate_ids,
unary,
hidden,
anchors,
top_k,
)
expected = torch.empty_like(actual)
for step in range(steps):
pred = (
anchors[:, None].expand(-1, top_k)
if step == 0
else candidate_ids[:, step - 1]
)
expected[:, step] = unary[:, step, None] + torch.einsum(
"bpr,bcr->bpc",
predecessors[pred] * hidden[:, step, None],
successors[candidate_ids[:, step]],
)
torch.testing.assert_close(actual, expected)
def _stub_base(monkeypatch, draft_logits):
"""A DFlashSpeculator.__init__ that allocates only what the base class would.
The real base class fills draft_logits from draft_logits_spec, so callers
pass a tensor already in that state.
"""
def init_base(self, _vllm_config, device):
self.draft_model_config = SimpleNamespace(
hf_config=SimpleNamespace(dflash_config={"selector_top_k": 3})
)
self.max_num_reqs = 2
self.num_query_per_req = 5
self.num_speculative_steps = 4
self.vocab_size = 17
self.draft_tokens = torch.empty((2, 4), dtype=torch.int64, device=device)
self.draft_logits = draft_logits
monkeypatch.setattr(DFlashSpeculator, "__init__", init_base)
def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch):
"""Greedy is the default, and it caches no proposal distribution.
The base class allocates draft_logits only for "probabilistic"; verification
reads `draft_logits is None` to decide whether a distribution is on offer, so
allocating one here would claim a proposal the walk never sampled from.
"""
_stub_base(monkeypatch, None)
speculator = DFlash2Speculator(None, torch.device("cpu"))
assert speculator.draft_logits is None
def test_selector_asks_for_fp32_proposal_logits():
"""The spec the base class allocates from: fp32, filled -inf.
Not the head dtype -- rounding selector scores to bf16 moves the argmax of a
candidate row often enough that the walk and the rejection sampler checking it
would no longer read the same distribution.
"""
dtype, fill = DFlash2Speculator.draft_logits_spec(None, None)
assert dtype is torch.float32
assert fill == float("-inf")
+25 -1
View File
@@ -21,12 +21,13 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
)
def _config(num_hidden_layers, layer_types=None, causal_override=None):
def _config(num_hidden_layers, layer_types=None, causal_override=None, is_causal=None):
dflash_config = None if causal_override is None else {"causal": causal_override}
return SimpleNamespace(
num_hidden_layers=num_hidden_layers,
layer_types=layer_types,
dflash_config=dflash_config,
is_causal=is_causal,
)
@@ -40,6 +41,19 @@ def _config(num_hidden_layers, layer_types=None, causal_override=None):
_config(2, layer_types=["sliding_attention"] * 2, causal_override=False),
True,
),
# DFlash2 stores the explicit attention semantics at the top level.
(
_config(
2,
layer_types=["sliding_attention"] * 2,
is_causal=False,
),
True,
),
(
_config(2, layer_types=["full_attention"] * 2, is_causal=True),
False,
),
# SWA-derived: full-attention layers are non-causal.
(_config(2, layer_types=["sliding_attention", "full_attention"]), True),
# SWA-derived: all-sliding is fully causal.
@@ -59,6 +73,16 @@ def test_dflash_layer_causal_is_per_layer():
assert _dflash_layer_causal(config, 1) is False
def test_dflash_layer_causal_honors_top_level_override():
config = _config(
2,
layer_types=["sliding_attention", "full_attention"],
is_causal=False,
)
assert _dflash_layer_causal(config, 0) is False
assert _dflash_layer_causal(config, 1) is False
def _vllm_config(**draft_config):
config = SimpleNamespace(**draft_config)
return SimpleNamespace(