[Perf] Add fused DFlash2 grouped convolution (#55960)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Misha Goin
2026-09-17 21:38:59 +00:00
committed by GitHub
parent 668d6c3a77
commit db7a24c230
4 changed files with 199 additions and 0 deletions
+124
View File
@@ -11,6 +11,8 @@ from vllm.config import CacheConfig, VllmConfig
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import direct_register_custom_op
from .qwen3_dflash import (
DFlashQwen3DecoderLayer,
@@ -20,6 +22,125 @@ from .qwen3_dflash import (
from .utils import maybe_prefix
@triton.jit
def _dflash2_grouped_conv_kernel(
x_ptr,
delta_ptr,
base_ptr,
output_ptr,
x_stride_row,
delta_stride_row,
delta_stride_tap,
base_stride_tap,
output_stride_row,
NUM_CHANNELS: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
GROUP_SIZE: tl.constexpr,
TAPS: tl.constexpr,
ELEMENT_BLOCK: tl.constexpr,
) -> None:
row = tl.program_id(0) // triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK)
col_block = tl.program_id(0) % triton.cdiv(NUM_CHANNELS, ELEMENT_BLOCK)
channels = col_block * ELEMENT_BLOCK + tl.arange(0, ELEMENT_BLOCK)
mask = channels < NUM_CHANNELS
groups = channels // GROUP_SIZE
position = row % BLOCK_SIZE
delta_row = delta_ptr + row * delta_stride_row
x_row = x_ptr + row * x_stride_row
accumulator = (
tl.load(base_ptr + channels, mask=mask, other=0.0).to(tl.float32)
+ tl.load(delta_row + groups, mask=mask, other=0.0).to(tl.float32)
) * tl.load(x_row + channels, mask=mask, other=0.0).to(tl.float32)
for tap in tl.static_range(1, TAPS):
tap_mask = mask & (position >= tap)
coefficient = tl.load(
base_ptr + tap * base_stride_tap + channels,
mask=tap_mask,
other=0.0,
).to(tl.float32) + tl.load(
delta_row + tap * delta_stride_tap + groups,
mask=tap_mask,
other=0.0,
).to(tl.float32)
x = tl.load(
x_ptr + (row - tap) * x_stride_row + channels,
mask=tap_mask,
other=0.0,
).to(tl.float32)
accumulator += coefficient * x
tl.store(
output_ptr + row * output_stride_row + channels,
accumulator,
mask=mask,
)
def dflash2_grouped_conv_impl(
x: torch.Tensor,
delta: torch.Tensor,
base: torch.Tensor,
block_size: int,
group_size: int,
) -> torch.Tensor:
"""DFlash2 grouped convolution adapted from LightSeek TokenSpeed PR #1399."""
num_rows, num_channels = x.shape
output = torch.empty_like(x)
if num_rows == 0:
return output
element_block = 1024 if num_rows >= 128 and num_channels % 1024 == 0 else 512
grid = (num_rows * triton.cdiv(num_channels, element_block),)
_dflash2_grouped_conv_kernel[grid](
x,
delta,
base,
output,
x.stride(0),
delta.stride(0),
delta.stride(1),
base.stride(0),
output.stride(0),
NUM_CHANNELS=num_channels,
BLOCK_SIZE=block_size,
GROUP_SIZE=group_size,
TAPS=base.shape[0],
ELEMENT_BLOCK=element_block,
num_warps=4,
)
return output
def dflash2_grouped_conv_fake(
x: torch.Tensor,
delta: torch.Tensor,
base: torch.Tensor,
block_size: int,
group_size: int,
) -> torch.Tensor:
return torch.empty_like(x)
direct_register_custom_op(
op_name="dflash2_grouped_conv",
op_func=dflash2_grouped_conv_impl,
fake_impl=dflash2_grouped_conv_fake,
tags=(torch.Tag.needs_fixed_stride_order,),
)
def dflash2_grouped_conv(
x: torch.Tensor,
delta: torch.Tensor,
base: torch.Tensor,
block_size: int,
group_size: int,
) -> torch.Tensor:
return torch.ops.vllm.dflash2_grouped_conv(x, delta, base, block_size, group_size)
def _grouped_conv(
hidden_states: torch.Tensor,
delta: torch.Tensor,
@@ -29,6 +150,9 @@ def _grouped_conv(
group_size: int,
taps: int,
) -> torch.Tensor:
if hidden_states.is_cuda:
return dflash2_grouped_conv(hidden_states, delta, base, block_size, group_size)
blocks = hidden_states.unflatten(-1, (num_groups, group_size))
coefficients = base.view(1, taps, num_groups, group_size) + delta.unsqueeze(-1)
output = coefficients[:, 0] * blocks