[Kernel][Perf] Add fused CUDA post-conv MTP decode kernel for Qwen3.5 GDN (#51674)

Signed-off-by: Jie Fang <jief@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jie Fang
2026-08-14 11:44:45 +08:00
committed by GitHub
co-authored by OpenAI Codex Claude Fable 5
parent d18bb7b390
commit 1be3628367
12 changed files with 1456 additions and 7 deletions
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3-Next/Qwen3.5 model."""
import os
from typing import Literal
import torch
from einops import rearrange
from torch import nn
from vllm import _custom_ops as ops
from vllm import envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import (
@@ -84,6 +86,9 @@ if GDN_AITER_TRITON_AVAILABLE:
logger = init_logger(__name__)
MAX_FUSED_GDN_MTP_TOKENS = 8
FUSED_GDN_STATE_DTYPES = (torch.float32, torch.bfloat16)
def _resolve_gdn_prefill_backend(
vllm_config: VllmConfig,
@@ -484,12 +489,50 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
self.enable_packed_recurrent_decode = (
envs.VLLM_ENABLE_FLA_PACKED_RECURRENT_DECODE
)
self.gdn_decode_kernel = envs.VLLM_GDN_DECODE_KERNEL.strip().lower()
if self.gdn_decode_kernel == "cuda":
reason = self._fused_gdn_decode_unsupported_reason(vllm_config)
if reason is not None:
if "VLLM_GDN_DECODE_KERNEL" in os.environ:
raise ValueError(
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
)
logger.info_once(
"Falling back to the Triton GDN decode path: %s", reason
)
self.gdn_decode_kernel = "triton"
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
def _fused_gdn_decode_unsupported_reason(
self, vllm_config: VllmConfig
) -> str | None:
conv_state_dtype, recurrent_state_dtype = self.get_state_dtype()
if (
self.gqa_interleaved_layout
or self.head_k_dim != 128
or self.head_v_dim != 128
or self.norm.activation != "silu"
or vllm_config.model_config.dtype != torch.bfloat16
or conv_state_dtype != torch.bfloat16
or recurrent_state_dtype not in FUSED_GDN_STATE_DTYPES
or not current_platform.has_device_capability(80)
):
return (
"the fused CUDA kernel requires a BF16 GDN model with "
"K=V=128, SiLU gating, non-interleaved GQA layout, BF16 "
"convolution cache, BF16 or FP32 recurrent state, and a "
"GPU with compute capability 8.0+"
)
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
return "torch.ops._C.fused_gdn_decode_post_conv_mtp is not built"
return None
def create_qkvz_proj(
self,
hidden_size: int,
@@ -846,6 +889,26 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
ba, _ = self.in_proj_ba(hidden_states)
use_fused_gdn_decode = (
self.enable_fused_gdn_decode
and hidden_states.dtype == torch.bfloat16
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
)
if use_fused_gdn_decode:
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
device=hidden_states.device,
)
torch.ops.vllm.qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz,
ba,
core_attn_out,
layer_name=_encode_layer_name(self.prefix),
)
output, _ = self.out_proj(core_attn_out.flatten(-2))
return output
if self.gqa_interleaved_layout:
# Qwen3-Next: unpack the interleaved GQA layout
query, key, value, z, b, a = self.fix_query_key_value_ordering(
@@ -862,8 +925,6 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
z = z.reshape(z.size(0), -1, self.head_v_dim)
b, a = self.split_ba(ba)
b = b.contiguous()
a = a.contiguous()
# ============================================================
# Part 2: Core Attention (Custom Op)
@@ -878,8 +939,8 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
torch.ops.vllm.qwen_gdn_attention_core(
mixed_qkv,
b,
a,
b.contiguous(),
a.contiguous(),
core_attn_out,
layer_name=_encode_layer_name(self.prefix),
)
@@ -1622,6 +1683,211 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
)
return
def _forward_core_decode_spec_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
attn_metadata: GDNAttentionMetadata,
) -> None:
state_indices = attn_metadata.spec_state_indices_tensor
cu_seqlens = attn_metadata.spec_query_start_loc
num_accepted_tokens = attn_metadata.num_accepted_tokens
assert state_indices is not None
assert cu_seqlens is not None
assert num_accepted_tokens is not None
num_requests = attn_metadata.num_spec_decodes
num_actual_tokens = attn_metadata.num_actual_tokens
conv_state = (
self.kv_cache[0]
if is_conv_state_dim_first()
else self.kv_cache[0].transpose(-1, -2)
)
conv_weights = self.conv1d.weight.view(
self.conv1d.weight.size(0), self.conv1d.weight.size(2)
)
mixed_qkv = causal_conv1d_update(
mixed_qkv[:num_actual_tokens],
conv_state,
conv_weights,
self.conv1d.bias,
self.activation,
conv_state_indices=state_indices[:num_requests, 0],
num_accepted_tokens=num_accepted_tokens[:num_requests],
query_start_loc=cu_seqlens[: num_requests + 1],
max_query_len=state_indices.size(1),
validate_data=False,
)
self._forward_core_decode_spec_post_conv_fused_norm(
mixed_qkv=mixed_qkv,
b=b[:num_actual_tokens],
a=a[:num_actual_tokens],
output_gate=output_gate[:num_actual_tokens],
core_attn_out=core_attn_out[:num_actual_tokens],
attn_metadata=attn_metadata,
)
def _forward_core_decode_spec_post_conv_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
attn_metadata: GDNAttentionMetadata,
) -> None:
state_indices = attn_metadata.spec_state_indices_tensor
cu_seqlens = attn_metadata.spec_query_start_loc
num_accepted_tokens = attn_metadata.num_accepted_tokens
assert state_indices is not None
assert cu_seqlens is not None
assert num_accepted_tokens is not None
num_requests = attn_metadata.num_spec_decodes
ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=self.A_log,
dt_bias=self.dt_bias,
state_indices=state_indices[:num_requests],
cu_seqlens=cu_seqlens[: num_requests + 1],
num_accepted_tokens=num_accepted_tokens[:num_requests],
state=self.kv_cache[1],
output_gate=output_gate,
norm_weight=self.norm.weight,
out=core_attn_out,
scale=self.head_k_dim**-0.5,
norm_eps=self.layer_norm_epsilon,
)
def _forward_core_fused_norm_packed(
self,
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
if attn_metadata_raw is None:
self._warmup_prefill_kernels(mixed_qkvz[:, :qkv_size], 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
mixed_qkv, output_gate_flat = mixed_qkvz.split(
[qkv_size, self.value_dim // self.tp_size], dim=-1
)
output_gate = output_gate_flat.reshape(
output_gate_flat.size(0), -1, self.head_v_dim
)
b, a = self.split_ba(ba)
self._forward_core_fused_norm(
mixed_qkv=mixed_qkv,
b=b,
a=a,
output_gate=output_gate,
core_attn_out=core_attn_out,
)
def _can_use_fused_gdn_mtp_decode(
self, attn_metadata: GDNAttentionMetadata
) -> bool:
state_indices = attn_metadata.spec_state_indices_tensor
return (
attn_metadata.spec_sequence_masks is not None
and attn_metadata.num_decodes == 0
and attn_metadata.num_spec_decodes > 0
and self.kv_cache[1].dtype in FUSED_GDN_STATE_DTYPES
and self.gdn_decode_kernel == "cuda"
and self.num_v_heads == 8 * self.num_k_heads
and state_indices is not None
and state_indices.size(1) <= MAX_FUSED_GDN_MTP_TOKENS
and hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp")
)
def _rms_norm_gated_cuda(
self,
x: torch.Tensor,
output_gate: torch.Tensor,
out: torch.Tensor,
) -> None:
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
layer_norm_fwd,
)
x_shape = x.shape
assert output_gate.shape == x_shape
assert out.shape == x_shape
x_2d = x.reshape(-1, x_shape[-1])
output_gate_2d = output_gate.reshape(-1, x_shape[-1])
out_2d = out.reshape(-1, x_shape[-1])
assert x_2d.stride(-1) == 1
assert output_gate_2d.stride(-1) == 1
assert out_2d.stride(-1) == 1
layer_norm_fwd(
x_2d,
self.norm.weight.contiguous(),
self.norm.bias,
self.norm.eps,
z=output_gate_2d,
out=out_2d,
group_size=(
x_shape[-1] if self.norm.group_size is None else self.norm.group_size
),
norm_before_gate=self.norm.norm_before_gate,
is_rms_norm=True,
activation=self.norm.activation,
)
def _forward_core_fused_norm(
self,
mixed_qkv: torch.Tensor,
b: torch.Tensor,
a: torch.Tensor,
output_gate: torch.Tensor,
core_attn_out: torch.Tensor,
) -> None:
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if attn_metadata_raw is None:
self._warmup_prefill_kernels(mixed_qkv, 0)
return
assert isinstance(attn_metadata_raw, dict)
attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index]
assert isinstance(attn_metadata, GDNAttentionMetadata)
if (
self._can_use_fused_gdn_mtp_decode(attn_metadata)
and attn_metadata.num_prefills == 0
):
self._forward_core_decode_spec_fused_norm(
mixed_qkv=mixed_qkv,
b=b,
a=a,
output_gate=output_gate,
core_attn_out=core_attn_out,
attn_metadata=attn_metadata,
)
return
self._forward_core(
mixed_qkv=mixed_qkv,
b=b.contiguous(),
a=a.contiguous(),
core_attn_out=core_attn_out,
)
num_actual_tokens = attn_metadata.num_actual_tokens
self._rms_norm_gated_cuda(
core_attn_out[:num_actual_tokens],
output_gate[:num_actual_tokens],
core_attn_out[:num_actual_tokens],
)
def qwen_gdn_attention_core(
qkv_or_qkvz: torch.Tensor,
@@ -1683,6 +1949,39 @@ direct_register_custom_op(
)
def qwen_gdn_attention_core_fused_norm_packed(
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: LayerNameType,
) -> None:
layer_name = _resolve_layer_name(layer_name)
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._forward_core_fused_norm_packed(
mixed_qkvz=mixed_qkvz,
ba=ba,
core_attn_out=core_attn_out,
)
def gdn_attention_core_fused_norm_packed_fake(
mixed_qkvz: torch.Tensor,
ba: torch.Tensor,
core_attn_out: torch.Tensor,
layer_name: LayerNameType,
) -> None:
return
direct_register_custom_op(
op_name="qwen_gdn_attention_core_fused_norm_packed",
op_func=qwen_gdn_attention_core_fused_norm_packed,
mutates_args=["core_attn_out"],
fake_impl=gdn_attention_core_fused_norm_packed_fake,
)
@triton.jit
def fused_gdn_gating_kernel(
g,