[Perf] Add fused DFlash2 grouped convolution (#55960)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-17 21:38:59 +00:00
committed by GitHub
parent 668d6c3a77
commit db7a24c230
4 changed files with 199 additions and 0 deletions
@@ -26,6 +26,8 @@ class DFlashCorrectnessConfig:
use_chat_completions: bool = False
enforce_eager: bool = False
disable_flashinfer_sampler: bool = False
language_model_only: bool = False
chat_template_kwargs: dict[str, object] | None = None
QWEN3_DFLASH = DFlashCorrectnessConfig(
@@ -35,6 +37,18 @@ QWEN3_DFLASH = DFlashCorrectnessConfig(
expected_acceptance_len=3.5,
)
QWEN3_8_DFLASH2_NVFP4 = DFlashCorrectnessConfig(
model="nvidia/Qwen3.8-27B-NVFP4",
draft_model="z-lab/Qwen3.8-27B-DFlash2",
expected_accuracy=0.90,
expected_acceptance_len=6.14 * 0.95,
num_speculative_tokens=7,
max_num_seqs=32,
use_chat_completions=True,
language_model_only=True,
chat_template_kwargs={"enable_thinking": False},
)
LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig(
model="poolside/Laguna-XS-2.1-NVFP4",
draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
@@ -104,6 +118,11 @@ def test_dflash_reference_acceptance_lengths(
True,
id="laguna-nvfp4-mrv2",
),
pytest.param(
QWEN3_8_DFLASH2_NVFP4,
True,
id="qwen3.8-dflash2-nvfp4-mrv2",
),
],
)
def test_dflash_correctness(
@@ -134,12 +153,14 @@ def test_dflash_correctness(
disable_log_stats=False,
enable_chunked_prefill=None,
compilation_config=CompilationConfig(),
language_model_only=config.language_model_only,
) as spec_runner:
spec_llm = spec_runner.llm
results = evaluate_gsm8k_offline(
spec_llm,
num_questions=config.num_questions,
use_chat_completions=config.use_chat_completions,
chat_template_kwargs=config.chat_template_kwargs,
)
accuracy = results["accuracy"]
acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
+50
View File
@@ -7,6 +7,7 @@ import pytest
import torch
from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges
from vllm.platforms import current_platform
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
@@ -35,6 +36,55 @@ def test_grouped_conv_matches_reference(block_size: int):
torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2))
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize(
"batch,num_groups,group_size,block_size,taps",
[
(7, 13, 11, 5, 1),
(7, 13, 11, 5, 3),
(7, 13, 11, 8, 2),
(16, 64, 16, 8, 2),
(16, 160, 16, 8, 2),
],
)
def test_grouped_conv_triton_matches_reference(
dtype: torch.dtype,
batch: int,
num_groups: int,
group_size: int,
block_size: int,
taps: int,
):
torch.manual_seed(0)
rows = batch * block_size
hidden = torch.randn(rows, num_groups * group_size, device="cuda", dtype=dtype)
base = torch.randn(taps, num_groups * group_size, device="cuda", dtype=dtype)
projected = torch.randn(rows, 2, taps, num_groups, device="cuda", dtype=dtype)
delta = projected[:, 1]
actual = _grouped_conv(
hidden, delta, base, block_size, num_groups, group_size, taps
)
hidden_blocks = hidden.float().view(batch, block_size, num_groups, group_size)
expected = torch.zeros_like(hidden_blocks)
base_blocks = base.float().view(taps, num_groups, group_size)
delta_blocks = delta.float().view(batch, block_size, taps, num_groups)
for position in range(block_size):
for tap in range(min(taps, position + 1)):
expected[:, position] += (
base_blocks[tap] + delta_blocks[:, position, tap, :, None]
) * hidden_blocks[:, position - tap]
torch.testing.assert_close(
actual,
expected.flatten(0, 1).flatten(-2).to(dtype),
rtol=1e-2 if dtype is torch.bfloat16 else 1e-5,
atol=1e-2 if dtype is torch.bfloat16 else 1e-5,
)
def test_selector_edges_match_sequential_reference():
torch.manual_seed(1)
batch, steps, top_k, rank = 2, 4, 3, 5