diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index c06642ea1a..fdb12efe11 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -283,6 +283,8 @@ steps: - vllm/v1/spec_decode/ - vllm/v1/worker/gpu/spec_decode/ - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py - vllm/model_executor/models/laguna_dflash.py - tests/v1/e2e/spec_decode/ commands: @@ -293,6 +295,8 @@ steps: dind: false device: mi300_1 timeout_in_minutes: 120 + commands: + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2" depends_on: - image-build-amd source_file_dependencies: diff --git a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py index 2b1800f782..48dcde2bbc 100644 --- a/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py +++ b/tests/v1/e2e/spec_decode/acceptance_rates/dflash/test_dflash.py @@ -26,6 +26,8 @@ class DFlashCorrectnessConfig: use_chat_completions: bool = False enforce_eager: bool = False disable_flashinfer_sampler: bool = False + language_model_only: bool = False + chat_template_kwargs: dict[str, object] | None = None QWEN3_DFLASH = DFlashCorrectnessConfig( @@ -35,6 +37,18 @@ QWEN3_DFLASH = DFlashCorrectnessConfig( expected_acceptance_len=3.5, ) +QWEN3_8_DFLASH2_NVFP4 = DFlashCorrectnessConfig( + model="nvidia/Qwen3.8-27B-NVFP4", + draft_model="z-lab/Qwen3.8-27B-DFlash2", + expected_accuracy=0.90, + expected_acceptance_len=6.14 * 0.95, + num_speculative_tokens=7, + max_num_seqs=32, + use_chat_completions=True, + language_model_only=True, + chat_template_kwargs={"enable_thinking": False}, +) + LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig( model="poolside/Laguna-XS-2.1-NVFP4", draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4", @@ -104,6 +118,11 @@ def test_dflash_reference_acceptance_lengths( True, id="laguna-nvfp4-mrv2", ), + pytest.param( + QWEN3_8_DFLASH2_NVFP4, + True, + id="qwen3.8-dflash2-nvfp4-mrv2", + ), ], ) def test_dflash_correctness( @@ -134,12 +153,14 @@ def test_dflash_correctness( disable_log_stats=False, enable_chunked_prefill=None, compilation_config=CompilationConfig(), + language_model_only=config.language_model_only, ) as spec_runner: spec_llm = spec_runner.llm results = evaluate_gsm8k_offline( spec_llm, num_questions=config.num_questions, use_chat_completions=config.use_chat_completions, + chat_template_kwargs=config.chat_template_kwargs, ) accuracy = results["accuracy"] acceptance_len = compute_acceptance_len(spec_llm.get_metrics()) diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index e7ff2b6d5d..72379fa7d7 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -7,6 +7,7 @@ import pytest import torch from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges +from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator @@ -35,6 +36,55 @@ def test_grouped_conv_matches_reference(block_size: int): torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2)) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA") +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize( + "batch,num_groups,group_size,block_size,taps", + [ + (7, 13, 11, 5, 1), + (7, 13, 11, 5, 3), + (7, 13, 11, 8, 2), + (16, 64, 16, 8, 2), + (16, 160, 16, 8, 2), + ], +) +def test_grouped_conv_triton_matches_reference( + dtype: torch.dtype, + batch: int, + num_groups: int, + group_size: int, + block_size: int, + taps: int, +): + torch.manual_seed(0) + rows = batch * block_size + hidden = torch.randn(rows, num_groups * group_size, device="cuda", dtype=dtype) + base = torch.randn(taps, num_groups * group_size, device="cuda", dtype=dtype) + projected = torch.randn(rows, 2, taps, num_groups, device="cuda", dtype=dtype) + delta = projected[:, 1] + + actual = _grouped_conv( + hidden, delta, base, block_size, num_groups, group_size, taps + ) + + hidden_blocks = hidden.float().view(batch, block_size, num_groups, group_size) + expected = torch.zeros_like(hidden_blocks) + base_blocks = base.float().view(taps, num_groups, group_size) + delta_blocks = delta.float().view(batch, block_size, taps, num_groups) + for position in range(block_size): + for tap in range(min(taps, position + 1)): + expected[:, position] += ( + base_blocks[tap] + delta_blocks[:, position, tap, :, None] + ) * hidden_blocks[:, position - tap] + + torch.testing.assert_close( + actual, + expected.flatten(0, 1).flatten(-2).to(dtype), + rtol=1e-2 if dtype is torch.bfloat16 else 1e-5, + atol=1e-2 if dtype is torch.bfloat16 else 1e-5, + ) + + def test_selector_edges_match_sequential_reference(): torch.manual_seed(1) batch, steps, top_k, rank = 2, 4, 3, 5 diff --git a/vllm/model_executor/models/qwen3_dflash2.py b/vllm/model_executor/models/qwen3_dflash2.py index cb027293f9..2043307d02 100644 --- a/vllm/model_executor/models/qwen3_dflash2.py +++ b/vllm/model_executor/models/qwen3_dflash2.py @@ -11,6 +11,8 @@ from vllm.config import CacheConfig, VllmConfig from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op from .qwen3_dflash import ( DFlashQwen3DecoderLayer, @@ -20,6 +22,125 @@ from .qwen3_dflash import ( from .utils import maybe_prefix +@triton.jit +def _dflash2_grouped_conv_kernel( + x_ptr, + delta_ptr, + base_ptr, + output_ptr, + x_stride_row, + delta_stride_row, + delta_stride_tap, + base_stride_tap, + output_stride_row, + NUM_CHANNELS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + GROUP_SIZE: tl.constexpr, + TAPS: tl.constexpr, + ELEMENT_BLOCK: tl.constexpr, +) -> None: + row = tl.program_id(0) // triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK) + col_block = tl.program_id(0) % triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK) + channels = col_block * ELEMENT_BLOCK + tl.arange(0, ELEMENT_BLOCK) + mask = channels < NUM_CHANNELS + groups = channels // GROUP_SIZE + position = row % BLOCK_SIZE + + delta_row = delta_ptr + row * delta_stride_row + x_row = x_ptr + row * x_stride_row + accumulator = ( + tl.load(base_ptr + channels, mask=mask, other=0.0).to(tl.float32) + + tl.load(delta_row + groups, mask=mask, other=0.0).to(tl.float32) + ) * tl.load(x_row + channels, mask=mask, other=0.0).to(tl.float32) + + for tap in tl.static_range(1, TAPS): + tap_mask = mask & (position >= tap) + coefficient = tl.load( + base_ptr + tap * base_stride_tap + channels, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + tl.load( + delta_row + tap * delta_stride_tap + groups, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + x = tl.load( + x_ptr + (row - tap) * x_stride_row + channels, + mask=tap_mask, + other=0.0, + ).to(tl.float32) + accumulator += coefficient * x + + tl.store( + output_ptr + row * output_stride_row + channels, + accumulator, + mask=mask, + ) + + +def dflash2_grouped_conv_impl( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + """DFlash2 grouped convolution adapted from LightSeek TokenSpeed PR #1399.""" + num_rows, num_channels = x.shape + output = torch.empty_like(x) + if num_rows == 0: + return output + + element_block = 1024 if num_rows >= 128 and num_channels % 1024 == 0 else 512 + grid = (num_rows * triton.cdiv(num_channels, element_block),) + _dflash2_grouped_conv_kernel[grid]( + x, + delta, + base, + output, + x.stride(0), + delta.stride(0), + delta.stride(1), + base.stride(0), + output.stride(0), + NUM_CHANNELS=num_channels, + BLOCK_SIZE=block_size, + GROUP_SIZE=group_size, + TAPS=base.shape[0], + ELEMENT_BLOCK=element_block, + num_warps=4, + ) + return output + + +def dflash2_grouped_conv_fake( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + return torch.empty_like(x) + + +direct_register_custom_op( + op_name="dflash2_grouped_conv", + op_func=dflash2_grouped_conv_impl, + fake_impl=dflash2_grouped_conv_fake, + tags=(torch.Tag.needs_fixed_stride_order,), +) + + +def dflash2_grouped_conv( + x: torch.Tensor, + delta: torch.Tensor, + base: torch.Tensor, + block_size: int, + group_size: int, +) -> torch.Tensor: + return torch.ops.vllm.dflash2_grouped_conv(x, delta, base, block_size, group_size) + + def _grouped_conv( hidden_states: torch.Tensor, delta: torch.Tensor, @@ -29,6 +150,9 @@ def _grouped_conv( group_size: int, taps: int, ) -> torch.Tensor: + if hidden_states.is_cuda: + return dflash2_grouped_conv(hidden_states, delta, base, block_size, group_size) + blocks = hidden_states.unflatten(-1, (num_groups, group_size)) coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1) output = coefficients[:, 0] * blocks