[Perf] Add fused DFlash2 grouped convolution (#55960)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -283,6 +283,8 @@ steps:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- vllm/model_executor/models/qwen3_dflash.py
|
||||
- vllm/model_executor/models/qwen3_dflash2.py
|
||||
- vllm/model_executor/models/qwen3_5.py
|
||||
- vllm/model_executor/models/laguna_dflash.py
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
@@ -293,6 +295,8 @@ steps:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "not dflash2"
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -26,6 +26,8 @@ class DFlashCorrectnessConfig:
|
||||
use_chat_completions: bool = False
|
||||
enforce_eager: bool = False
|
||||
disable_flashinfer_sampler: bool = False
|
||||
language_model_only: bool = False
|
||||
chat_template_kwargs: dict[str, object] | None = None
|
||||
|
||||
|
||||
QWEN3_DFLASH = DFlashCorrectnessConfig(
|
||||
@@ -35,6 +37,18 @@ QWEN3_DFLASH = DFlashCorrectnessConfig(
|
||||
expected_acceptance_len=3.5,
|
||||
)
|
||||
|
||||
QWEN3_8_DFLASH2_NVFP4 = DFlashCorrectnessConfig(
|
||||
model="nvidia/Qwen3.8-27B-NVFP4",
|
||||
draft_model="z-lab/Qwen3.8-27B-DFlash2",
|
||||
expected_accuracy=0.90,
|
||||
expected_acceptance_len=6.14 * 0.95,
|
||||
num_speculative_tokens=7,
|
||||
max_num_seqs=32,
|
||||
use_chat_completions=True,
|
||||
language_model_only=True,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
LAGUNA_DFLASH_NVFP4 = DFlashCorrectnessConfig(
|
||||
model="poolside/Laguna-XS-2.1-NVFP4",
|
||||
draft_model="poolside/Laguna-XS-2.1-DFlash-NVFP4",
|
||||
@@ -104,6 +118,11 @@ def test_dflash_reference_acceptance_lengths(
|
||||
True,
|
||||
id="laguna-nvfp4-mrv2",
|
||||
),
|
||||
pytest.param(
|
||||
QWEN3_8_DFLASH2_NVFP4,
|
||||
True,
|
||||
id="qwen3.8-dflash2-nvfp4-mrv2",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_dflash_correctness(
|
||||
@@ -134,12 +153,14 @@ def test_dflash_correctness(
|
||||
disable_log_stats=False,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config=CompilationConfig(),
|
||||
language_model_only=config.language_model_only,
|
||||
) as spec_runner:
|
||||
spec_llm = spec_runner.llm
|
||||
results = evaluate_gsm8k_offline(
|
||||
spec_llm,
|
||||
num_questions=config.num_questions,
|
||||
use_chat_completions=config.use_chat_completions,
|
||||
chat_template_kwargs=config.chat_template_kwargs,
|
||||
)
|
||||
accuracy = results["accuracy"]
|
||||
acceptance_len = compute_acceptance_len(spec_llm.get_metrics())
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.qwen3_dflash2 import _grouped_conv, _score_edges
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator
|
||||
from vllm.v1.worker.gpu.spec_decode.dflash2.speculator import DFlash2Speculator
|
||||
|
||||
@@ -35,6 +36,55 @@ def test_grouped_conv_matches_reference(block_size: int):
|
||||
torch.testing.assert_close(actual, expected.flatten(0, 1).flatten(-2))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"batch,num_groups,group_size,block_size,taps",
|
||||
[
|
||||
(7, 13, 11, 5, 1),
|
||||
(7, 13, 11, 5, 3),
|
||||
(7, 13, 11, 8, 2),
|
||||
(16, 64, 16, 8, 2),
|
||||
(16, 160, 16, 8, 2),
|
||||
],
|
||||
)
|
||||
def test_grouped_conv_triton_matches_reference(
|
||||
dtype: torch.dtype,
|
||||
batch: int,
|
||||
num_groups: int,
|
||||
group_size: int,
|
||||
block_size: int,
|
||||
taps: int,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
rows = batch * block_size
|
||||
hidden = torch.randn(rows, num_groups * group_size, device="cuda", dtype=dtype)
|
||||
base = torch.randn(taps, num_groups * group_size, device="cuda", dtype=dtype)
|
||||
projected = torch.randn(rows, 2, taps, num_groups, device="cuda", dtype=dtype)
|
||||
delta = projected[:, 1]
|
||||
|
||||
actual = _grouped_conv(
|
||||
hidden, delta, base, block_size, num_groups, group_size, taps
|
||||
)
|
||||
|
||||
hidden_blocks = hidden.float().view(batch, block_size, num_groups, group_size)
|
||||
expected = torch.zeros_like(hidden_blocks)
|
||||
base_blocks = base.float().view(taps, num_groups, group_size)
|
||||
delta_blocks = delta.float().view(batch, block_size, taps, num_groups)
|
||||
for position in range(block_size):
|
||||
for tap in range(min(taps, position + 1)):
|
||||
expected[:, position] += (
|
||||
base_blocks[tap] + delta_blocks[:, position, tap, :, None]
|
||||
) * hidden_blocks[:, position - tap]
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual,
|
||||
expected.flatten(0, 1).flatten(-2).to(dtype),
|
||||
rtol=1e-2 if dtype is torch.bfloat16 else 1e-5,
|
||||
atol=1e-2 if dtype is torch.bfloat16 else 1e-5,
|
||||
)
|
||||
|
||||
|
||||
def test_selector_edges_match_sequential_reference():
|
||||
torch.manual_seed(1)
|
||||
batch, steps, top_k, rank = 2, 4, 3, 5
|
||||
|
||||
@@ -11,6 +11,8 @@ from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from .qwen3_dflash import (
|
||||
DFlashQwen3DecoderLayer,
|
||||
@@ -20,6 +22,125 @@ from .qwen3_dflash import (
|
||||
from .utils import maybe_prefix
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dflash2_grouped_conv_kernel(
|
||||
x_ptr,
|
||||
delta_ptr,
|
||||
base_ptr,
|
||||
output_ptr,
|
||||
x_stride_row,
|
||||
delta_stride_row,
|
||||
delta_stride_tap,
|
||||
base_stride_tap,
|
||||
output_stride_row,
|
||||
NUM_CHANNELS: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
GROUP_SIZE: tl.constexpr,
|
||||
TAPS: tl.constexpr,
|
||||
ELEMENT_BLOCK: tl.constexpr,
|
||||
) -> None:
|
||||
row = tl.program_id(0) // triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK)
|
||||
col_block = tl.program_id(0) % triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK)
|
||||
channels = col_block * ELEMENT_BLOCK + tl.arange(0, ELEMENT_BLOCK)
|
||||
mask = channels < NUM_CHANNELS
|
||||
groups = channels // GROUP_SIZE
|
||||
position = row % BLOCK_SIZE
|
||||
|
||||
delta_row = delta_ptr + row * delta_stride_row
|
||||
x_row = x_ptr + row * x_stride_row
|
||||
accumulator = (
|
||||
tl.load(base_ptr + channels, mask=mask, other=0.0).to(tl.float32)
|
||||
+ tl.load(delta_row + groups, mask=mask, other=0.0).to(tl.float32)
|
||||
) * tl.load(x_row + channels, mask=mask, other=0.0).to(tl.float32)
|
||||
|
||||
for tap in tl.static_range(1, TAPS):
|
||||
tap_mask = mask & (position >= tap)
|
||||
coefficient = tl.load(
|
||||
base_ptr + tap * base_stride_tap + channels,
|
||||
mask=tap_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32) + tl.load(
|
||||
delta_row + tap * delta_stride_tap + groups,
|
||||
mask=tap_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
x = tl.load(
|
||||
x_ptr + (row - tap) * x_stride_row + channels,
|
||||
mask=tap_mask,
|
||||
other=0.0,
|
||||
).to(tl.float32)
|
||||
accumulator += coefficient * x
|
||||
|
||||
tl.store(
|
||||
output_ptr + row * output_stride_row + channels,
|
||||
accumulator,
|
||||
mask=mask,
|
||||
)
|
||||
|
||||
|
||||
def dflash2_grouped_conv_impl(
|
||||
x: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
base: torch.Tensor,
|
||||
block_size: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""DFlash2 grouped convolution adapted from LightSeek TokenSpeed PR #1399."""
|
||||
num_rows, num_channels = x.shape
|
||||
output = torch.empty_like(x)
|
||||
if num_rows == 0:
|
||||
return output
|
||||
|
||||
element_block = 1024 if num_rows >= 128 and num_channels % 1024 == 0 else 512
|
||||
grid = (num_rows * triton.cdiv(num_channels, element_block),)
|
||||
_dflash2_grouped_conv_kernel[grid](
|
||||
x,
|
||||
delta,
|
||||
base,
|
||||
output,
|
||||
x.stride(0),
|
||||
delta.stride(0),
|
||||
delta.stride(1),
|
||||
base.stride(0),
|
||||
output.stride(0),
|
||||
NUM_CHANNELS=num_channels,
|
||||
BLOCK_SIZE=block_size,
|
||||
GROUP_SIZE=group_size,
|
||||
TAPS=base.shape[0],
|
||||
ELEMENT_BLOCK=element_block,
|
||||
num_warps=4,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def dflash2_grouped_conv_fake(
|
||||
x: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
base: torch.Tensor,
|
||||
block_size: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="dflash2_grouped_conv",
|
||||
op_func=dflash2_grouped_conv_impl,
|
||||
fake_impl=dflash2_grouped_conv_fake,
|
||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||
)
|
||||
|
||||
|
||||
def dflash2_grouped_conv(
|
||||
x: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
base: torch.Tensor,
|
||||
block_size: int,
|
||||
group_size: int,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.dflash2_grouped_conv(x, delta, base, block_size, group_size)
|
||||
|
||||
|
||||
def _grouped_conv(
|
||||
hidden_states: torch.Tensor,
|
||||
delta: torch.Tensor,
|
||||
@@ -29,6 +150,9 @@ def _grouped_conv(
|
||||
group_size: int,
|
||||
taps: int,
|
||||
) -> torch.Tensor:
|
||||
if hidden_states.is_cuda:
|
||||
return dflash2_grouped_conv(hidden_states, delta, base, block_size, group_size)
|
||||
|
||||
blocks = hidden_states.unflatten(-1, (num_groups, group_size))
|
||||
coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1)
|
||||
output = coefficients[:, 0] * blocks
|
||||
|
||||
Reference in New Issue
Block a user