[CI] Split (B200) Miscellaneous Kernels into mHC, FLA Ops and Misc named jobs (#58609)

Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Thang Nguyen
2026-09-26 04:10:14 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Kevin H. Luu
parent a4eb3f25d6
commit 2f41e00235
11 changed files with 31 additions and 6 deletions
View File
@@ -0,0 +1,553 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
LayerNormFwdKernel,
calc_rows_per_block,
layer_norm_fwd,
layernorm_fn,
rms_norm_ref,
)
from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
def layer_norm_ref(
x,
weight,
bias,
z=None,
eps=1e-6,
group_size=None,
norm_before_gate=True,
is_rms_norm=False,
):
"""Reference implementation for both layer norm and RMS norm."""
if is_rms_norm:
# Use the imported rms_norm_ref for RMS norm cases
return rms_norm_ref(
x,
weight,
bias,
z=z,
eps=eps,
group_size=group_size,
norm_before_gate=norm_before_gate,
upcast=True,
)
# Layer norm implementation
dtype = x.dtype
x = x.float()
weight = weight.float()
bias = bias.float() if bias is not None else None
z = z.float() if z is not None else None
if z is not None and not norm_before_gate:
x = x * F.silu(z)
if group_size is None:
# Layer norm: subtract mean
mean = x.mean(dim=-1, keepdim=True)
var = ((x - mean).square()).mean(dim=-1, keepdim=True)
rstd = 1 / torch.sqrt(var + eps)
out = (x - mean) * rstd * weight
if bias is not None:
out = out + bias
else:
# Group norm
from einops import rearrange
x_group = rearrange(x, "... (g d) -> ... g d", d=group_size)
mean = x_group.mean(dim=-1, keepdim=True)
var = ((x_group - mean).square()).mean(dim=-1, keepdim=True)
rstd = 1 / torch.sqrt(var + eps)
x_group = (x_group - mean) * rstd
out = rearrange(x_group, "... g d -> ... (g d)") * weight
if bias is not None:
out = out + bias
if z is not None and norm_before_gate:
out *= F.silu(z)
return out.to(dtype)
DTYPES = [torch.float16, torch.bfloat16, torch.float32]
# Test various M sizes to ensure rows_per_block logic works correctly
NUM_TOKENS = [
1,
7,
16,
63,
128,
256,
512,
1024,
2048,
4096,
5789,
8189,
8191,
16383,
32767,
]
HIDDEN_SIZES = [64, 128, 256, 1024]
GROUP_SIZES = [None, 64, 128] # None means full hidden size
NORM_BEFORE_GATE = [True, False]
IS_RMS_NORM = [True, False]
SEEDS = [0, 42]
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
rows_per_token: int,
) -> None:
device = torch.device(DEVICE)
group_size = 128
max_num_tokens = 512
kernel = LayerNormFwdKernel()
warmup_keys = set(
kernel.get_warmup_keys(
max_num_tokens=max_num_tokens,
rows_per_token=rows_per_token,
group_size=group_size,
x_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
device=device,
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
)
for num_tokens in range(1, max_num_tokens + 1):
num_rows = num_tokens * rows_per_token
runtime_key = kernel.dispatch(
x_dtype=torch.bfloat16,
y_dtype=torch.bfloat16,
weight_dtype=torch.bfloat16,
bias_dtype=None,
z_dtype=torch.bfloat16,
mean_dtype=None,
rstd_dtype=torch.float32,
x_aligned=True,
y_aligned=True,
weight_aligned=True,
bias_aligned=True,
z_aligned=True,
mean_aligned=True,
rstd_aligned=True,
stride_x_row=group_size,
stride_y_row=group_size,
stride_z_row=group_size,
M=num_rows,
N=group_size,
BLOCK_N=triton.next_power_of_2(group_size),
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
norm_before_gate=True,
is_rms_norm=True,
activation="silu",
)
assert runtime_key in warmup_keys
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
@torch.inference_mode()
def test_layer_norm_fwd_basic(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
seed: int,
is_rms_norm: bool,
) -> None:
"""Test basic layer norm forward pass without z (gate) tensor."""
set_random_seed(seed)
device = torch.device(DEVICE)
# Create inputs
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Run the triton kernel
out, mean, rstd = layer_norm_fwd(
x, weight, bias, eps, z=None, is_rms_norm=is_rms_norm
)
# Run reference implementation
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=is_rms_norm)
# Check outputs
assert out.shape == x.shape
assert out.dtype == x.dtype
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
# Check mean and rstd shapes
if not is_rms_norm:
assert mean.shape == (num_tokens,)
assert rstd.shape == (num_tokens,)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", [128, 256, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("norm_before_gate", NORM_BEFORE_GATE)
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
@torch.inference_mode()
def test_layer_norm_fwd_with_gate(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
norm_before_gate: bool,
is_rms_norm: bool,
) -> None:
"""Test layer norm forward pass with z (gate) tensor."""
set_random_seed(42)
device = torch.device(DEVICE)
# Create inputs
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
z = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Run the triton kernel
out, mean, rstd = layer_norm_fwd(
x,
weight,
bias,
eps,
z=z,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
)
# Run reference implementation
ref_out = layer_norm_ref(
x,
weight,
bias,
z=z,
eps=eps,
norm_before_gate=norm_before_gate,
is_rms_norm=is_rms_norm,
)
# Check outputs
assert out.shape == x.shape
assert out.dtype == x.dtype
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("num_tokens", [128, 512])
@pytest.mark.parametrize("hidden_size", [512, 1024])
@pytest.mark.parametrize("group_size", [64, 128, 256])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
@torch.inference_mode()
def test_layer_norm_fwd_with_groups(
num_tokens: int,
hidden_size: int,
group_size: int,
dtype: torch.dtype,
is_rms_norm: bool,
) -> None:
"""Test layer norm forward pass with group normalization."""
if hidden_size % group_size != 0:
pytest.skip(
f"hidden_size {hidden_size} not divisible by group_size {group_size}"
)
set_random_seed(42)
device = torch.device(DEVICE)
# Create inputs
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
ngroups = hidden_size // group_size
# Run the triton kernel
out, mean, rstd = layer_norm_fwd(
x, weight, bias, eps, z=None, group_size=group_size, is_rms_norm=is_rms_norm
)
# Run reference implementation
ref_out = layer_norm_ref(
x, weight, bias, z=None, eps=eps, group_size=group_size, is_rms_norm=is_rms_norm
)
# Check outputs
assert out.shape == x.shape
assert out.dtype == x.dtype
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
# Check mean and rstd shapes for groups
if not is_rms_norm:
assert mean.shape == (ngroups * num_tokens,)
assert rstd.shape == (ngroups * num_tokens,)
@pytest.mark.parametrize("num_tokens", [7, 63, 128, 513, 1024, 2049])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@torch.inference_mode()
def test_layer_norm_rows_per_block(
num_tokens: int,
dtype: torch.dtype,
) -> None:
"""Test that rows_per_block logic works correctly for various M sizes."""
set_random_seed(42)
device = torch.device(DEVICE)
hidden_size = 1024
# Create inputs
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Run the triton kernel
out, mean, rstd = layer_norm_fwd(x, weight, bias, eps, z=None, is_rms_norm=False)
# Run reference implementation
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
# Check outputs
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_strided_input(dtype: torch.dtype) -> None:
"""Test that the kernel handles non-contiguous (strided)
inputs correctly."""
set_random_seed(42)
device = torch.device(DEVICE)
num_tokens = 128
hidden_size = 1024
# Create a larger tensor and take a strided slice
x_large = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
x = x_large[:, :hidden_size]
# Make it contiguous for the kernel
x_contiguous = x.contiguous()
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Run the triton kernel with contiguous input
out, mean, rstd = layer_norm_fwd(
x_contiguous, weight, bias, eps, z=None, is_rms_norm=False
)
# Run reference implementation
ref_out = layer_norm_ref(
x_contiguous, weight, bias, z=None, eps=eps, is_rms_norm=False
)
# Check outputs
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("num_tokens", [1, 128, 2048])
@pytest.mark.parametrize("hidden_size", [768, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@torch.inference_mode()
def test_output_buffer_provided(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
) -> None:
"""Test that the kernel works when an output buffer is provided."""
set_random_seed(42)
device = torch.device(DEVICE)
# Create inputs
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Pre-allocate output buffer
out_buffer = torch.empty_like(x)
# Run the triton kernel with provided output
out, mean, rstd = layer_norm_fwd(
x, weight, bias, eps, z=None, out=out_buffer, is_rms_norm=False
)
# Check that the provided buffer was used
assert out.data_ptr() == out_buffer.data_ptr()
# Run reference implementation
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
# Check outputs
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize(
"shape",
[
(4, 16, 1024), # 3D tensor
(2, 8, 512, 256), # 4D tensor
],
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@torch.inference_mode()
def test_multidimensional_input(
shape: tuple,
dtype: torch.dtype,
) -> None:
"""Test that the autograd function handles multidimensional inputs."""
set_random_seed(42)
device = torch.device(DEVICE)
hidden_size = shape[-1]
# Create inputs
x = torch.randn(*shape, dtype=dtype, device=device)
weight = torch.randn(hidden_size, dtype=dtype, device=device)
bias = torch.randn(hidden_size, dtype=dtype, device=device)
eps = 1e-6
# Run through autograd function
out = layernorm_fn(x, weight, bias, z=None, eps=eps)
# Run reference implementation
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
# Check outputs
assert out.shape == x.shape
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("num_tokens", [1, 128, 1024])
@pytest.mark.parametrize("hidden_size", [64, 256, 1024])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("has_gate", [True, False])
@pytest.mark.parametrize("group_size", [None, 64])
@pytest.mark.parametrize("norm_before_gate", [True, False])
@torch.inference_mode()
def test_rmsnorm_gated_forward_native_dtype(
default_vllm_config,
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
has_gate: bool,
group_size: int | None,
norm_before_gate: bool,
):
"""Test that RMSNormGated.forward_native preserves input dtype."""
if group_size is not None and hidden_size % group_size != 0:
pytest.skip(
f"hidden_size {hidden_size} not divisible by group_size {group_size}"
)
from vllm.model_executor.layers.layernorm import RMSNormGated
device = torch.device(DEVICE)
set_random_seed(42)
layer = RMSNormGated(
hidden_size,
eps=1e-5,
group_size=group_size,
norm_before_gate=norm_before_gate,
device=device,
dtype=dtype,
)
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
z = (
torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
if has_gate
else None
)
out = layer.forward_native(x, z)
# Verify dtype preservation
assert out.dtype == dtype, f"Expected {dtype}, got {out.dtype}"
# Verify numerical correctness against reference
ref_out = rms_norm_ref(
x,
layer.weight,
layer.bias,
z=z,
eps=1e-5,
group_size=group_size,
norm_before_gate=norm_before_gate,
upcast=True,
)
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_rmsnorm_gated_3d_matches_flattened_2d(
default_vllm_config,
dtype: torch.dtype,
) -> None:
"""Qwen GDN now norms (N, H, D) in place of a flatten to (N*H, D).
Last-dim RMS + gate must match the flattened layout on both the native
and CUDA/HIP kernels, including a 3D z view that is not contiguous.
"""
from vllm.model_executor.layers.layernorm import RMSNormGated
set_random_seed(42)
device = torch.device(DEVICE)
num_tokens, num_heads, head_dim = 5, 4, 64
layer = RMSNormGated(
head_dim,
eps=1e-5,
group_size=None,
norm_before_gate=True,
device=device,
dtype=dtype,
)
packed = torch.randn(num_tokens, num_heads, 2, head_dim, dtype=dtype, device=device)
x3 = packed[:, :, 0, :]
z3 = packed[:, :, 1, :]
assert not x3.is_contiguous()
assert not z3.is_contiguous()
x2 = x3.reshape(-1, head_dim).contiguous()
z2 = z3.reshape(-1, head_dim).contiguous()
native_3d = layer.forward_native(x3, z3)
native_2d = layer.forward_native(x2, z2).reshape(num_tokens, num_heads, head_dim)
torch.testing.assert_close(native_3d, native_2d, atol=0, rtol=0)
cuda_3d = layer.forward_cuda(x3, z3)
cuda_2d = layer.forward_cuda(x2, z2).reshape(num_tokens, num_heads, head_dim)
torch.testing.assert_close(cuda_3d, cuda_2d, atol=1e-3, rtol=1e-2)
torch.testing.assert_close(cuda_3d, native_3d, atol=1e-2, rtol=1e-2)
assert cuda_3d.shape == (num_tokens, num_heads, head_dim)
if __name__ == "__main__":
# Run a quick smoke test
test_layer_norm_fwd_basic(128, 1024, torch.float16, 42, False)
test_layer_norm_fwd_with_gate(128, 1024, torch.float16, True, False)
test_layer_norm_rows_per_block(513, torch.float16)
print("All smoke tests passed!")
@@ -0,0 +1,389 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GDN post-convolution kernels.
The prefill tests cover preparation and the MTP tests cover recurrent state
updates plus output normalization and gating.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm import _custom_ops as ops
from vllm.third_party.flash_linear_attention.ops import (
fused_sigmoid_gating_delta_rule_update,
)
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import (
fused_post_conv_prep,
)
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
rmsnorm_fn,
)
def reference_post_conv(
conv_output: torch.Tensor,
a: torch.Tensor,
b: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
H: int,
K: int,
V: int,
apply_l2norm: bool = True,
output_g_exp: bool = False,
):
"""Reference implementation using individual ops."""
L = conv_output.shape[0]
HV = A_log.shape[0]
# Split
q_flat, k_flat, v_flat = torch.split(conv_output, [H * K, H * K, HV * V], dim=-1)
# Rearrange + contiguous
q = q_flat.view(L, H, K).contiguous()
k = k_flat.view(L, H, K).contiguous()
v = v_flat.view(L, HV, V).contiguous()
# L2 norm
if apply_l2norm:
q = F.normalize(q.float(), p=2, dim=-1, eps=1e-6).to(conv_output.dtype)
k = F.normalize(k.float(), p=2, dim=-1, eps=1e-6).to(conv_output.dtype)
# Gating
x = a.float() + dt_bias.float()
sp = F.softplus(x, beta=1.0, threshold=20.0)
g = -torch.exp(A_log.float()) * sp
if output_g_exp:
g = torch.exp(g)
beta_out = torch.sigmoid(b.float())
return q, k, v, g, beta_out
# Qwen3.5-35B config: H=16, HV=32, K=128, V=128
# Qwen3.5-397B config: H=16, HV=64, K=128, V=128
@pytest.mark.parametrize(
"H, HV, K, V",
[
(16, 32, 128, 128), # 35B
(16, 64, 128, 128), # 397B
(4, 8, 64, 64), # small
],
)
@pytest.mark.parametrize("L", [1, 16, 128, 512, 2048])
@pytest.mark.parametrize("apply_l2norm", [True, False])
@pytest.mark.parametrize("output_g_exp", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
def test_fused_post_conv_correctness(H, HV, K, V, L, apply_l2norm, output_g_exp, dtype):
"""Test fused kernel matches reference for all configs."""
torch.manual_seed(42)
device = "cuda"
qkv_dim = 2 * H * K + HV * V
conv_output = torch.randn(L, qkv_dim, dtype=dtype, device=device)
a = torch.randn(L, HV, dtype=dtype, device=device)
b = torch.randn(L, HV, dtype=dtype, device=device)
A_log = torch.randn(HV, dtype=torch.float32, device=device) - 2.0
dt_bias = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
# Reference
ref_q, ref_k, ref_v, ref_g, ref_beta = reference_post_conv(
conv_output,
a,
b,
A_log,
dt_bias,
H,
K,
V,
apply_l2norm,
output_g_exp,
)
# Fused kernel
fused_q, fused_k, fused_v, fused_g, fused_beta = fused_post_conv_prep(
conv_output,
a,
b,
A_log,
dt_bias,
num_k_heads=H,
head_k_dim=K,
head_v_dim=V,
apply_l2norm=apply_l2norm,
output_g_exp=output_g_exp,
)
# Check shapes
assert fused_q.shape == (L, H, K), f"q shape: {fused_q.shape}"
assert fused_k.shape == (L, H, K), f"k shape: {fused_k.shape}"
assert fused_v.shape == (L, HV, V), f"v shape: {fused_v.shape}"
assert fused_g.shape == (L, HV), f"g shape: {fused_g.shape}"
assert fused_beta.shape == (L, HV), f"beta shape: {fused_beta.shape}"
# Check dtypes
assert fused_q.dtype == dtype
assert fused_k.dtype == dtype
assert fused_v.dtype == dtype
assert fused_g.dtype == torch.float32
assert fused_beta.dtype == torch.float32
# Check contiguity
assert fused_q.is_contiguous()
assert fused_k.is_contiguous()
assert fused_v.is_contiguous()
# Check values
atol_qkv = 1e-2 if apply_l2norm else 1e-3
rtol_qkv = 1e-2 if apply_l2norm else 1e-3
torch.testing.assert_close(fused_q, ref_q, atol=atol_qkv, rtol=rtol_qkv)
torch.testing.assert_close(fused_k, ref_k, atol=atol_qkv, rtol=rtol_qkv)
torch.testing.assert_close(fused_v, ref_v, atol=1e-3, rtol=1e-3)
torch.testing.assert_close(fused_g, ref_g, atol=1e-4, rtol=1e-4)
torch.testing.assert_close(fused_beta, ref_beta, atol=1e-4, rtol=1e-4)
@pytest.mark.parametrize("L", [1, 64, 256])
def test_fused_post_conv_sanity(L):
"""Sanity checks: no NaN, unit-norm q/k, beta in (0,1)."""
torch.manual_seed(0)
device = "cuda"
H, HV, K, V = 16, 32, 128, 128
qkv_dim = 2 * H * K + HV * V
conv_output = torch.randn(L, qkv_dim, dtype=torch.bfloat16, device=device)
a = torch.randn(L, HV, dtype=torch.bfloat16, device=device)
b = torch.randn(L, HV, dtype=torch.bfloat16, device=device)
A_log = torch.randn(HV, dtype=torch.float32, device=device) - 2.0
dt_bias = torch.randn(HV, dtype=torch.float32, device=device)
q, k, v, g, beta = fused_post_conv_prep(
conv_output,
a,
b,
A_log,
dt_bias,
num_k_heads=H,
head_k_dim=K,
head_v_dim=V,
)
# Basic sanity
assert not torch.isnan(q).any(), "NaN in q"
assert not torch.isnan(k).any(), "NaN in k"
assert not torch.isnan(v).any(), "NaN in v"
assert not torch.isnan(g).any(), "NaN in g"
assert not torch.isnan(beta).any(), "NaN in beta"
# L2 norm check: each head vector should have unit norm
q_norms = torch.norm(q.float(), dim=-1)
k_norms = torch.norm(k.float(), dim=-1)
torch.testing.assert_close(q_norms, torch.ones_like(q_norms), atol=1e-3, rtol=1e-3)
torch.testing.assert_close(k_norms, torch.ones_like(k_norms), atol=1e-3, rtol=1e-3)
# Beta should be in (0, 1)
assert (beta >= 0).all() and (beta <= 1).all(), "beta out of range"
def test_fused_post_conv_l0():
"""Test L=0 edge case."""
device = "cuda"
H, HV, K, V = 16, 32, 128, 128
qkv_dim = 2 * H * K + HV * V
conv_output = torch.empty(0, qkv_dim, dtype=torch.bfloat16, device=device)
a = torch.empty(0, HV, dtype=torch.bfloat16, device=device)
b = torch.empty(0, HV, dtype=torch.bfloat16, device=device)
A_log = torch.randn(HV, dtype=torch.float32, device=device)
dt_bias = torch.randn(HV, dtype=torch.float32, device=device)
q, k, v, g, beta = fused_post_conv_prep(
conv_output,
a,
b,
A_log,
dt_bias,
num_k_heads=H,
head_k_dim=K,
head_v_dim=V,
)
assert q.shape == (0, H, K)
assert g.shape == (0, HV)
@pytest.mark.parametrize(
"head_ratio,tp_size,query_lengths,state_dtype,norm_dtype",
[
pytest.param(8, 16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
pytest.param(8, 4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
pytest.param(8, 16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
pytest.param(8, 4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
pytest.param(8, 16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
pytest.param(8, 4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
pytest.param(
1,
1,
(4, 4),
torch.float32,
torch.bfloat16,
id="ratio1-tp1-fp32",
),
pytest.param(
2,
1,
(4, 2, 0),
torch.bfloat16,
torch.bfloat16,
id="ratio2-tp1-ragged-bf16",
),
pytest.param(
2,
4,
(8,),
torch.float32,
torch.float32,
id="ratio2-tp4-max-fp32",
),
pytest.param(
3,
2,
(4, 2, 0),
torch.float32,
torch.float32,
id="ratio3-tp2-ragged-fp32",
),
pytest.param(
4,
4,
(8,),
torch.float32,
torch.bfloat16,
id="ratio4-tp4-max-fp32",
),
],
)
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
@torch.inference_mode()
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
head_ratio: int,
tp_size: int,
query_lengths: tuple[int, ...],
state_dtype: torch.dtype,
norm_dtype: torch.dtype,
output_gate_activation: str,
) -> None:
if torch.cuda.get_device_capability() < (8, 0):
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
pytest.skip("fused GDN decode MTP op is not built")
torch.manual_seed(0)
device = "cuda"
H = 16 // tp_size
HV = head_ratio * H
K = V = 128
num_reqs = len(query_lengths)
state_width = max(query_lengths)
num_tokens = sum(query_lengths)
num_slots = num_reqs * state_width + 1
scale = K**-0.5
eps = 1e-6
mixed_qkv = torch.randn(
num_tokens,
2 * H * K + HV * V,
dtype=torch.bfloat16,
device=device,
)
query, key, value = torch.split(
mixed_qkv,
[H * K, H * K, HV * V],
dim=-1,
)
query = query.view(1, num_tokens, H, K)
key = key.view(1, num_tokens, H, K)
value = value.view(1, num_tokens, HV, V)
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
b, a = ba.chunk(2, dim=-1)
assert not a.is_contiguous()
assert not b.is_contiguous()
A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device)
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device)
norm_weight = torch.randn(V, dtype=norm_dtype, device=device)
state_ref = (
0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device)
).to(state_dtype)
state_actual = state_ref.clone()
state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view(
num_reqs, state_width
)
cu_seqlens = torch.tensor(
[0, *torch.tensor(query_lengths).cumsum(0).tolist()],
dtype=torch.int32,
device=device,
)
num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device)
if query_lengths[-1] == 0:
state_indices[-1].zero_()
for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)):
num_accepted_tokens.fill_(accepted_tokens)
if query_lengths[-1] == 0:
num_accepted_tokens[-1] = 1
raw_ref, _ = fused_sigmoid_gating_delta_rule_update(
A_log=A_log,
a=a,
b=b,
dt_bias=dt_bias,
q=query,
k=key,
v=value,
initial_state=state_ref,
inplace_final_state=True,
cu_seqlens=cu_seqlens,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
scale=scale,
use_qk_l2norm_in_kernel=True,
)
expected = rmsnorm_fn(
raw_ref.squeeze(0),
norm_weight,
None,
z=output_gate,
eps=eps,
norm_before_gate=True,
activation=output_gate_activation,
)
actual = ops.fused_gdn_decode_post_conv_mtp(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=A_log,
dt_bias=dt_bias,
state_indices=state_indices,
cu_seqlens=cu_seqlens,
num_accepted_tokens=num_accepted_tokens,
state=state_actual,
output_gate=output_gate,
norm_weight=norm_weight,
out=torch.empty_like(output_gate),
scale=scale,
norm_eps=eps,
output_gate_activation=output_gate_activation,
)
output_error = (actual.float() - expected.float()).norm()
output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20)
assert output_relative_l2 < 5e-4, (
f"MTP output relative L2 mismatch at step {step}: "
f"{output_relative_l2.item():.6g}"
)
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
fused_recurrent_gated_delta_rule,
fused_recurrent_gated_delta_rule_packed_decode,
)
DEVICE = current_platform.device_type
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.",
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
def test_fused_recurrent_packed_decode_matches_reference(
dtype: torch.dtype, strided_mixed_qkv: bool
):
torch.manual_seed(0)
# Small but representative GDN config (Qwen3Next defaults are K=128, V=128).
B = 32
H = 4
HV = 8 # grouped value attention: HV must be divisible by H
K = 128
V = 128
qkv_dim = 2 * (H * K) + (HV * V)
device = torch.device(DEVICE)
if strided_mixed_qkv:
# Simulate a packed view into a larger projection buffer:
# mixed_qkv.stride(0) > mixed_qkv.shape[1]
proj = torch.randn((B, qkv_dim + 64), device=device, dtype=dtype)
mixed_qkv = proj[:, :qkv_dim]
else:
mixed_qkv = torch.randn((B, qkv_dim), device=device, dtype=dtype)
a = torch.randn((B, HV), device=device, dtype=dtype)
b = torch.randn((B, HV), device=device, dtype=dtype)
A_log = torch.randn((HV,), device=device, dtype=dtype)
dt_bias = torch.randn((HV,), device=device, dtype=dtype)
# Continuous batching indices (include PAD_SLOT_ID=-1 cases). Index 0 is
# reserved as NULL_BLOCK_ID (CUDA graph padding), so valid slots start at 1.
ssm_state_indices = torch.arange(1, B + 1, device=device, dtype=torch.int32)
ssm_state_indices[-3:] = -1
state0 = torch.randn((B + 1, HV, V, K), device=device, dtype=dtype)
state_ref = state0.clone()
state_packed = state0.clone()
out_packed = torch.empty((B, 1, HV, V), device=device, dtype=dtype)
# Reference path: materialize contiguous Q/K/V + explicit gating.
q, k, v = torch.split(mixed_qkv, [H * K, H * K, HV * V], dim=-1)
q = q.view(B, H, K).unsqueeze(1).contiguous()
k = k.view(B, H, K).unsqueeze(1).contiguous()
v = v.view(B, HV, V).unsqueeze(1).contiguous()
x = a.float() + dt_bias.float()
softplus_x = torch.where(
x <= 20.0, torch.log1p(torch.exp(torch.clamp(x, max=20.0))), x
)
g = (-torch.exp(A_log.float()) * softplus_x).unsqueeze(1)
beta = torch.sigmoid(b.float()).unsqueeze(1)
out_ref, state_ref = fused_recurrent_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
scale=K**-0.5,
initial_state=state_ref,
inplace_final_state=True,
cu_seqlens=None,
ssm_state_indices=ssm_state_indices,
use_qk_l2norm_in_kernel=True,
)
# Packed path: fused gating + recurrent directly from packed mixed_qkv.
fused_recurrent_gated_delta_rule_packed_decode(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=A_log,
dt_bias=dt_bias,
scale=K**-0.5,
initial_state=state_packed,
out=out_packed,
ssm_state_indices=ssm_state_indices,
use_qk_l2norm_in_kernel=True,
)
atol = 2e-2 if dtype != torch.float32 else 1e-4
rtol = 1e-2 if dtype != torch.float32 else 1e-4
# Output rows for PAD_SLOT_ID entries are never written (uninitialized in
# both paths), so compare only the valid rows.
valid = ssm_state_indices > 0
torch.testing.assert_close(out_packed[valid], out_ref[valid], rtol=rtol, atol=atol)
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
def test_packed_decode_keeps_beta_in_fp32():
device = torch.device(DEVICE)
dtype = torch.bfloat16
mixed_qkv = torch.ones((1, 3), device=device, dtype=dtype)
a = torch.zeros((1, 1), device=device, dtype=dtype)
b = torch.full((1, 1), 0.5, device=device, dtype=dtype)
params = torch.zeros((1,), device=device, dtype=dtype)
ssm_state_indices = torch.ones((1,), device=device, dtype=torch.int32)
state_packed = torch.zeros((2, 1, 1, 1), device=device, dtype=torch.float32)
out_packed = torch.empty((1, 1, 1, 1), device=device, dtype=dtype)
fused_recurrent_gated_delta_rule_packed_decode(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=params,
dt_bias=params,
scale=1.0,
initial_state=state_packed,
out=out_packed,
ssm_state_indices=ssm_state_indices,
)
expected_beta = torch.sigmoid(b.float()).squeeze()
torch.testing.assert_close(
state_packed[1, 0, 0, 0], expected_beta, rtol=1e-6, atol=1e-6
)
def test_packed_decode_supports_large_batch_head_grid():
B, H, HV, K, V = 1024, 8, 64, 1, 1
device = torch.device(DEVICE)
gates = torch.empty((B, HV), device=device)
params = torch.empty((HV,), device=device)
out = torch.empty((B, 1, HV, V), device=device)
fused_recurrent_gated_delta_rule_packed_decode(
mixed_qkv=torch.empty((B, 2 * H * K + HV * V), device=device),
a=gates,
b=gates,
A_log=params,
dt_bias=params,
scale=1.0,
initial_state=torch.empty((1, HV, V, K), device=device),
out=out,
ssm_state_indices=torch.zeros((B,), device=device, dtype=torch.int32),
)
assert torch.count_nonzero(out).item() == 0
@@ -0,0 +1,199 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops import (
fused_recurrent_gated_delta_rule,
fused_sigmoid_gating_delta_rule_update,
)
from vllm.utils.torch_utils import set_random_seed
DEVICE = current_platform.device_type
@pytest.mark.parametrize("tp_size", [1])
@pytest.mark.parametrize("num_reqs", [1, 2, 4])
@pytest.mark.parametrize("num_k_heads", [16])
@pytest.mark.parametrize("num_v_heads", [32])
@pytest.mark.parametrize("head_k_dim", [128])
@pytest.mark.parametrize("head_v_dim", [128])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_fused_sigmoid_gating_delta_rule_update_non_spec(
tp_size: int,
num_reqs: int,
num_k_heads: int,
num_v_heads: int,
head_k_dim: int,
head_v_dim: int,
dtype: torch.dtype,
) -> None:
torch.set_default_device(DEVICE)
set_random_seed(0)
key_dim = head_k_dim * num_k_heads
value_dim = head_v_dim * num_v_heads
mixed_qkv_dim = (key_dim * 2 + value_dim) // tp_size
seq_len = 1 # seq_len is 1 for decode
num_tokens = num_reqs * seq_len
total_entries = num_tokens * 2
mixed_qkv = torch.rand(num_tokens, mixed_qkv_dim, dtype=dtype)
query, key, value = torch.split(
mixed_qkv,
[
key_dim // tp_size,
key_dim // tp_size,
value_dim // tp_size,
],
dim=-1,
)
query = query.view(1, num_tokens, num_k_heads, head_k_dim)
key = key.view(1, num_tokens, num_k_heads, head_k_dim)
value = value.view(1, num_tokens, num_v_heads, head_v_dim)
A_log = torch.rand(num_v_heads // tp_size, dtype=dtype)
dt_bias = torch.rand(num_v_heads // tp_size, dtype=dtype)
a = torch.rand(num_tokens, num_v_heads, dtype=dtype)
b = torch.rand(num_tokens, num_v_heads, dtype=dtype)
# Entry 0 is reserved as NULL_BLOCK_ID (CUDA graph padding), so valid
# state indices start at 1.
ssm_state = torch.rand(
total_entries + 1, num_v_heads, head_k_dim, head_v_dim, dtype=dtype
)
state_indices = (torch.randperm(total_entries, dtype=torch.int32) + 1)[:num_tokens]
cu_seqlens = torch.arange(0, num_tokens + 1, dtype=torch.int32)
beta = b.sigmoid()
g = -A_log.float().exp() * F.softplus(a.float() + dt_bias)
core_attn_out_ref, last_recurrent_state_ref = fused_recurrent_gated_delta_rule(
q=query,
k=key,
v=value,
g=g.unsqueeze(0),
beta=beta.unsqueeze(0),
initial_state=ssm_state.clone(),
inplace_final_state=True,
ssm_state_indices=state_indices,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=True,
)
core_attn_out, last_recurrent_state = fused_sigmoid_gating_delta_rule_update(
A_log=A_log,
a=a,
b=b,
dt_bias=dt_bias,
q=query,
k=key,
v=value,
initial_state=ssm_state,
inplace_final_state=True,
ssm_state_indices=state_indices,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=True,
)
torch.testing.assert_close(core_attn_out, core_attn_out_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
last_recurrent_state, last_recurrent_state_ref, atol=1e-2, rtol=1e-2
)
@pytest.mark.parametrize("tp_size", [1])
@pytest.mark.parametrize("num_reqs", [1, 2, 4])
@pytest.mark.parametrize("num_k_heads", [16])
@pytest.mark.parametrize("num_v_heads", [32])
@pytest.mark.parametrize("head_k_dim", [128])
@pytest.mark.parametrize("head_v_dim", [128])
@pytest.mark.parametrize("num_speculative_tokens", [1, 3])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_fused_sigmoid_gating_delta_rule_update_spec(
tp_size: int,
num_reqs: int,
num_k_heads: int,
num_v_heads: int,
head_k_dim: int,
head_v_dim: int,
num_speculative_tokens: int,
dtype: torch.dtype,
) -> None:
torch.set_default_device(DEVICE)
set_random_seed(0)
key_dim = head_k_dim * num_k_heads
value_dim = head_v_dim * num_v_heads
mixed_qkv_dim = (key_dim * 2 + value_dim) // tp_size
num_tokens = num_reqs * (num_speculative_tokens + 1)
total_entries = num_tokens * 2
mixed_qkv = torch.rand(num_tokens, mixed_qkv_dim, dtype=dtype)
query, key, value = torch.split(
mixed_qkv,
[
key_dim // tp_size,
key_dim // tp_size,
value_dim // tp_size,
],
dim=-1,
)
query = query.view(1, num_tokens, num_k_heads, head_k_dim)
key = key.view(1, num_tokens, num_k_heads, head_k_dim)
value = value.view(1, num_tokens, num_v_heads, head_v_dim)
A_log = torch.rand(num_v_heads // tp_size, dtype=dtype)
dt_bias = torch.rand(num_v_heads // tp_size, dtype=dtype)
a = torch.rand(num_tokens, num_v_heads, dtype=dtype)
b = torch.rand(num_tokens, num_v_heads, dtype=dtype)
# Entry 0 is reserved as NULL_BLOCK_ID (CUDA graph padding), so valid
# state indices start at 1.
ssm_state = torch.rand(
total_entries + 1, num_v_heads, head_k_dim, head_v_dim, dtype=dtype
)
state_indices = (torch.randperm(total_entries, dtype=torch.int32) + 1)[
:num_tokens
].view(num_reqs, num_speculative_tokens + 1)
num_accepted_tokens = torch.randint(
1, num_speculative_tokens + 1, (num_reqs,), dtype=torch.int32
)
cu_seqlens = torch.arange(
0, num_tokens + 1, num_speculative_tokens + 1, dtype=torch.int32
)
beta = b.sigmoid()
g = -A_log.float().exp() * F.softplus(a.float() + dt_bias)
core_attn_out_ref, last_recurrent_state_ref = fused_recurrent_gated_delta_rule(
q=query,
k=key,
v=value,
g=g.unsqueeze(0),
beta=beta.unsqueeze(0),
initial_state=ssm_state.clone(),
inplace_final_state=True,
ssm_state_indices=state_indices,
cu_seqlens=cu_seqlens,
num_accepted_tokens=num_accepted_tokens,
use_qk_l2norm_in_kernel=True,
)
core_attn_out, last_recurrent_state = fused_sigmoid_gating_delta_rule_update(
A_log=A_log,
a=a,
b=b,
dt_bias=dt_bias,
q=query,
k=key,
v=value,
initial_state=ssm_state,
inplace_final_state=True,
ssm_state_indices=state_indices,
cu_seqlens=cu_seqlens,
num_accepted_tokens=num_accepted_tokens,
use_qk_l2norm_in_kernel=True,
)
torch.testing.assert_close(core_attn_out, core_attn_out_ref, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(
last_recurrent_state, last_recurrent_state_ref, atol=1e-2, rtol=1e-2
)