[CI] Split (B200) Miscellaneous Kernels into mHC, FLA Ops and Misc named jobs (#58609)
Signed-off-by: Thang Nguyen <thangnguyenvn647@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Kevin H. Luu
parent
a4eb3f25d6
commit
2f41e00235
@@ -0,0 +1,553 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
LayerNormFwdKernel,
|
||||
calc_rows_per_block,
|
||||
layer_norm_fwd,
|
||||
layernorm_fn,
|
||||
rms_norm_ref,
|
||||
)
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "xpu" if current_platform.is_xpu() else "cuda"
|
||||
|
||||
|
||||
def layer_norm_ref(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
z=None,
|
||||
eps=1e-6,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=False,
|
||||
):
|
||||
"""Reference implementation for both layer norm and RMS norm."""
|
||||
if is_rms_norm:
|
||||
# Use the imported rms_norm_ref for RMS norm cases
|
||||
return rms_norm_ref(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
z=z,
|
||||
eps=eps,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
upcast=True,
|
||||
)
|
||||
|
||||
# Layer norm implementation
|
||||
dtype = x.dtype
|
||||
x = x.float()
|
||||
weight = weight.float()
|
||||
bias = bias.float() if bias is not None else None
|
||||
z = z.float() if z is not None else None
|
||||
|
||||
if z is not None and not norm_before_gate:
|
||||
x = x * F.silu(z)
|
||||
|
||||
if group_size is None:
|
||||
# Layer norm: subtract mean
|
||||
mean = x.mean(dim=-1, keepdim=True)
|
||||
var = ((x - mean).square()).mean(dim=-1, keepdim=True)
|
||||
rstd = 1 / torch.sqrt(var + eps)
|
||||
out = (x - mean) * rstd * weight
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
else:
|
||||
# Group norm
|
||||
from einops import rearrange
|
||||
|
||||
x_group = rearrange(x, "... (g d) -> ... g d", d=group_size)
|
||||
mean = x_group.mean(dim=-1, keepdim=True)
|
||||
var = ((x_group - mean).square()).mean(dim=-1, keepdim=True)
|
||||
rstd = 1 / torch.sqrt(var + eps)
|
||||
x_group = (x_group - mean) * rstd
|
||||
out = rearrange(x_group, "... g d -> ... (g d)") * weight
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
|
||||
if z is not None and norm_before_gate:
|
||||
out *= F.silu(z)
|
||||
|
||||
return out.to(dtype)
|
||||
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16, torch.float32]
|
||||
# Test various M sizes to ensure rows_per_block logic works correctly
|
||||
NUM_TOKENS = [
|
||||
1,
|
||||
7,
|
||||
16,
|
||||
63,
|
||||
128,
|
||||
256,
|
||||
512,
|
||||
1024,
|
||||
2048,
|
||||
4096,
|
||||
5789,
|
||||
8189,
|
||||
8191,
|
||||
16383,
|
||||
32767,
|
||||
]
|
||||
HIDDEN_SIZES = [64, 128, 256, 1024]
|
||||
GROUP_SIZES = [None, 64, 128] # None means full hidden size
|
||||
NORM_BEFORE_GATE = [True, False]
|
||||
IS_RMS_NORM = [True, False]
|
||||
SEEDS = [0, 42]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows_per_token", [1, 2, 4, 8, 16])
|
||||
def test_layer_norm_fwd_warmup_keys_cover_qwen_gdn(
|
||||
rows_per_token: int,
|
||||
) -> None:
|
||||
device = torch.device(DEVICE)
|
||||
group_size = 128
|
||||
max_num_tokens = 512
|
||||
kernel = LayerNormFwdKernel()
|
||||
warmup_keys = set(
|
||||
kernel.get_warmup_keys(
|
||||
max_num_tokens=max_num_tokens,
|
||||
rows_per_token=rows_per_token,
|
||||
group_size=group_size,
|
||||
x_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
device=device,
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
)
|
||||
|
||||
for num_tokens in range(1, max_num_tokens + 1):
|
||||
num_rows = num_tokens * rows_per_token
|
||||
runtime_key = kernel.dispatch(
|
||||
x_dtype=torch.bfloat16,
|
||||
y_dtype=torch.bfloat16,
|
||||
weight_dtype=torch.bfloat16,
|
||||
bias_dtype=None,
|
||||
z_dtype=torch.bfloat16,
|
||||
mean_dtype=None,
|
||||
rstd_dtype=torch.float32,
|
||||
x_aligned=True,
|
||||
y_aligned=True,
|
||||
weight_aligned=True,
|
||||
bias_aligned=True,
|
||||
z_aligned=True,
|
||||
mean_aligned=True,
|
||||
rstd_aligned=True,
|
||||
stride_x_row=group_size,
|
||||
stride_y_row=group_size,
|
||||
stride_z_row=group_size,
|
||||
M=num_rows,
|
||||
N=group_size,
|
||||
BLOCK_N=triton.next_power_of_2(group_size),
|
||||
ROWS_PER_BLOCK=calc_rows_per_block(num_rows, device),
|
||||
norm_before_gate=True,
|
||||
is_rms_norm=True,
|
||||
activation="silu",
|
||||
)
|
||||
assert runtime_key in warmup_keys
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
|
||||
@torch.inference_mode()
|
||||
def test_layer_norm_fwd_basic(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
is_rms_norm: bool,
|
||||
) -> None:
|
||||
"""Test basic layer norm forward pass without z (gate) tensor."""
|
||||
set_random_seed(seed)
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Run the triton kernel
|
||||
out, mean, rstd = layer_norm_fwd(
|
||||
x, weight, bias, eps, z=None, is_rms_norm=is_rms_norm
|
||||
)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=is_rms_norm)
|
||||
|
||||
# Check outputs
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Check mean and rstd shapes
|
||||
if not is_rms_norm:
|
||||
assert mean.shape == (num_tokens,)
|
||||
assert rstd.shape == (num_tokens,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", [128, 256, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("norm_before_gate", NORM_BEFORE_GATE)
|
||||
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
|
||||
@torch.inference_mode()
|
||||
def test_layer_norm_fwd_with_gate(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
norm_before_gate: bool,
|
||||
is_rms_norm: bool,
|
||||
) -> None:
|
||||
"""Test layer norm forward pass with z (gate) tensor."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
z = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Run the triton kernel
|
||||
out, mean, rstd = layer_norm_fwd(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
eps,
|
||||
z=z,
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(
|
||||
x,
|
||||
weight,
|
||||
bias,
|
||||
z=z,
|
||||
eps=eps,
|
||||
norm_before_gate=norm_before_gate,
|
||||
is_rms_norm=is_rms_norm,
|
||||
)
|
||||
|
||||
# Check outputs
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [128, 512])
|
||||
@pytest.mark.parametrize("hidden_size", [512, 1024])
|
||||
@pytest.mark.parametrize("group_size", [64, 128, 256])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("is_rms_norm", IS_RMS_NORM)
|
||||
@torch.inference_mode()
|
||||
def test_layer_norm_fwd_with_groups(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
group_size: int,
|
||||
dtype: torch.dtype,
|
||||
is_rms_norm: bool,
|
||||
) -> None:
|
||||
"""Test layer norm forward pass with group normalization."""
|
||||
if hidden_size % group_size != 0:
|
||||
pytest.skip(
|
||||
f"hidden_size {hidden_size} not divisible by group_size {group_size}"
|
||||
)
|
||||
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = None if is_rms_norm else torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
ngroups = hidden_size // group_size
|
||||
|
||||
# Run the triton kernel
|
||||
out, mean, rstd = layer_norm_fwd(
|
||||
x, weight, bias, eps, z=None, group_size=group_size, is_rms_norm=is_rms_norm
|
||||
)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(
|
||||
x, weight, bias, z=None, eps=eps, group_size=group_size, is_rms_norm=is_rms_norm
|
||||
)
|
||||
|
||||
# Check outputs
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == x.dtype
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Check mean and rstd shapes for groups
|
||||
if not is_rms_norm:
|
||||
assert mean.shape == (ngroups * num_tokens,)
|
||||
assert rstd.shape == (ngroups * num_tokens,)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [7, 63, 128, 513, 1024, 2049])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_layer_norm_rows_per_block(
|
||||
num_tokens: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test that rows_per_block logic works correctly for various M sizes."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
hidden_size = 1024
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Run the triton kernel
|
||||
out, mean, rstd = layer_norm_fwd(x, weight, bias, eps, z=None, is_rms_norm=False)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
|
||||
|
||||
# Check outputs
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_strided_input(dtype: torch.dtype) -> None:
|
||||
"""Test that the kernel handles non-contiguous (strided)
|
||||
inputs correctly."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
num_tokens = 128
|
||||
hidden_size = 1024
|
||||
|
||||
# Create a larger tensor and take a strided slice
|
||||
x_large = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
|
||||
x = x_large[:, :hidden_size]
|
||||
|
||||
# Make it contiguous for the kernel
|
||||
x_contiguous = x.contiguous()
|
||||
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Run the triton kernel with contiguous input
|
||||
out, mean, rstd = layer_norm_fwd(
|
||||
x_contiguous, weight, bias, eps, z=None, is_rms_norm=False
|
||||
)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(
|
||||
x_contiguous, weight, bias, z=None, eps=eps, is_rms_norm=False
|
||||
)
|
||||
|
||||
# Check outputs
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128, 2048])
|
||||
@pytest.mark.parametrize("hidden_size", [768, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_output_buffer_provided(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test that the kernel works when an output buffer is provided."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Pre-allocate output buffer
|
||||
out_buffer = torch.empty_like(x)
|
||||
|
||||
# Run the triton kernel with provided output
|
||||
out, mean, rstd = layer_norm_fwd(
|
||||
x, weight, bias, eps, z=None, out=out_buffer, is_rms_norm=False
|
||||
)
|
||||
|
||||
# Check that the provided buffer was used
|
||||
assert out.data_ptr() == out_buffer.data_ptr()
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
|
||||
|
||||
# Check outputs
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(4, 16, 1024), # 3D tensor
|
||||
(2, 8, 512, 256), # 4D tensor
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_multidimensional_input(
|
||||
shape: tuple,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Test that the autograd function handles multidimensional inputs."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
hidden_size = shape[-1]
|
||||
|
||||
# Create inputs
|
||||
x = torch.randn(*shape, dtype=dtype, device=device)
|
||||
weight = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
bias = torch.randn(hidden_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
# Run through autograd function
|
||||
out = layernorm_fn(x, weight, bias, z=None, eps=eps)
|
||||
|
||||
# Run reference implementation
|
||||
ref_out = layer_norm_ref(x, weight, bias, z=None, eps=eps, is_rms_norm=False)
|
||||
|
||||
# Check outputs
|
||||
assert out.shape == x.shape
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128, 1024])
|
||||
@pytest.mark.parametrize("hidden_size", [64, 256, 1024])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("has_gate", [True, False])
|
||||
@pytest.mark.parametrize("group_size", [None, 64])
|
||||
@pytest.mark.parametrize("norm_before_gate", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_rmsnorm_gated_forward_native_dtype(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
has_gate: bool,
|
||||
group_size: int | None,
|
||||
norm_before_gate: bool,
|
||||
):
|
||||
"""Test that RMSNormGated.forward_native preserves input dtype."""
|
||||
if group_size is not None and hidden_size % group_size != 0:
|
||||
pytest.skip(
|
||||
f"hidden_size {hidden_size} not divisible by group_size {group_size}"
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.layernorm import RMSNormGated
|
||||
|
||||
device = torch.device(DEVICE)
|
||||
set_random_seed(42)
|
||||
|
||||
layer = RMSNormGated(
|
||||
hidden_size,
|
||||
eps=1e-5,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
z = (
|
||||
torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
if has_gate
|
||||
else None
|
||||
)
|
||||
|
||||
out = layer.forward_native(x, z)
|
||||
|
||||
# Verify dtype preservation
|
||||
assert out.dtype == dtype, f"Expected {dtype}, got {out.dtype}"
|
||||
|
||||
# Verify numerical correctness against reference
|
||||
ref_out = rms_norm_ref(
|
||||
x,
|
||||
layer.weight,
|
||||
layer.bias,
|
||||
z=z,
|
||||
eps=1e-5,
|
||||
group_size=group_size,
|
||||
norm_before_gate=norm_before_gate,
|
||||
upcast=True,
|
||||
)
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_rmsnorm_gated_3d_matches_flattened_2d(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Qwen GDN now norms (N, H, D) in place of a flatten to (N*H, D).
|
||||
|
||||
Last-dim RMS + gate must match the flattened layout on both the native
|
||||
and CUDA/HIP kernels, including a 3D z view that is not contiguous.
|
||||
"""
|
||||
from vllm.model_executor.layers.layernorm import RMSNormGated
|
||||
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
num_tokens, num_heads, head_dim = 5, 4, 64
|
||||
layer = RMSNormGated(
|
||||
head_dim,
|
||||
eps=1e-5,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
packed = torch.randn(num_tokens, num_heads, 2, head_dim, dtype=dtype, device=device)
|
||||
x3 = packed[:, :, 0, :]
|
||||
z3 = packed[:, :, 1, :]
|
||||
assert not x3.is_contiguous()
|
||||
assert not z3.is_contiguous()
|
||||
x2 = x3.reshape(-1, head_dim).contiguous()
|
||||
z2 = z3.reshape(-1, head_dim).contiguous()
|
||||
|
||||
native_3d = layer.forward_native(x3, z3)
|
||||
native_2d = layer.forward_native(x2, z2).reshape(num_tokens, num_heads, head_dim)
|
||||
torch.testing.assert_close(native_3d, native_2d, atol=0, rtol=0)
|
||||
|
||||
cuda_3d = layer.forward_cuda(x3, z3)
|
||||
cuda_2d = layer.forward_cuda(x2, z2).reshape(num_tokens, num_heads, head_dim)
|
||||
torch.testing.assert_close(cuda_3d, cuda_2d, atol=1e-3, rtol=1e-2)
|
||||
torch.testing.assert_close(cuda_3d, native_3d, atol=1e-2, rtol=1e-2)
|
||||
assert cuda_3d.shape == (num_tokens, num_heads, head_dim)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run a quick smoke test
|
||||
test_layer_norm_fwd_basic(128, 1024, torch.float16, 42, False)
|
||||
test_layer_norm_fwd_with_gate(128, 1024, torch.float16, True, False)
|
||||
test_layer_norm_rows_per_block(513, torch.float16)
|
||||
print("All smoke tests passed!")
|
||||
@@ -0,0 +1,389 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for GDN post-convolution kernels.
|
||||
|
||||
The prefill tests cover preparation and the MTP tests cover recurrent state
|
||||
updates plus output normalization and gating.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_gdn_prefill_post_conv import (
|
||||
fused_post_conv_prep,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.layernorm_guard import (
|
||||
rmsnorm_fn,
|
||||
)
|
||||
|
||||
|
||||
def reference_post_conv(
|
||||
conv_output: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
H: int,
|
||||
K: int,
|
||||
V: int,
|
||||
apply_l2norm: bool = True,
|
||||
output_g_exp: bool = False,
|
||||
):
|
||||
"""Reference implementation using individual ops."""
|
||||
L = conv_output.shape[0]
|
||||
HV = A_log.shape[0]
|
||||
|
||||
# Split
|
||||
q_flat, k_flat, v_flat = torch.split(conv_output, [H * K, H * K, HV * V], dim=-1)
|
||||
|
||||
# Rearrange + contiguous
|
||||
q = q_flat.view(L, H, K).contiguous()
|
||||
k = k_flat.view(L, H, K).contiguous()
|
||||
v = v_flat.view(L, HV, V).contiguous()
|
||||
|
||||
# L2 norm
|
||||
if apply_l2norm:
|
||||
q = F.normalize(q.float(), p=2, dim=-1, eps=1e-6).to(conv_output.dtype)
|
||||
k = F.normalize(k.float(), p=2, dim=-1, eps=1e-6).to(conv_output.dtype)
|
||||
|
||||
# Gating
|
||||
x = a.float() + dt_bias.float()
|
||||
sp = F.softplus(x, beta=1.0, threshold=20.0)
|
||||
g = -torch.exp(A_log.float()) * sp
|
||||
|
||||
if output_g_exp:
|
||||
g = torch.exp(g)
|
||||
|
||||
beta_out = torch.sigmoid(b.float())
|
||||
|
||||
return q, k, v, g, beta_out
|
||||
|
||||
|
||||
# Qwen3.5-35B config: H=16, HV=32, K=128, V=128
|
||||
# Qwen3.5-397B config: H=16, HV=64, K=128, V=128
|
||||
@pytest.mark.parametrize(
|
||||
"H, HV, K, V",
|
||||
[
|
||||
(16, 32, 128, 128), # 35B
|
||||
(16, 64, 128, 128), # 397B
|
||||
(4, 8, 64, 64), # small
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("L", [1, 16, 128, 512, 2048])
|
||||
@pytest.mark.parametrize("apply_l2norm", [True, False])
|
||||
@pytest.mark.parametrize("output_g_exp", [True, False])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
def test_fused_post_conv_correctness(H, HV, K, V, L, apply_l2norm, output_g_exp, dtype):
|
||||
"""Test fused kernel matches reference for all configs."""
|
||||
torch.manual_seed(42)
|
||||
device = "cuda"
|
||||
qkv_dim = 2 * H * K + HV * V
|
||||
|
||||
conv_output = torch.randn(L, qkv_dim, dtype=dtype, device=device)
|
||||
a = torch.randn(L, HV, dtype=dtype, device=device)
|
||||
b = torch.randn(L, HV, dtype=dtype, device=device)
|
||||
A_log = torch.randn(HV, dtype=torch.float32, device=device) - 2.0
|
||||
dt_bias = torch.randn(HV, dtype=torch.float32, device=device) * 0.1
|
||||
|
||||
# Reference
|
||||
ref_q, ref_k, ref_v, ref_g, ref_beta = reference_post_conv(
|
||||
conv_output,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
H,
|
||||
K,
|
||||
V,
|
||||
apply_l2norm,
|
||||
output_g_exp,
|
||||
)
|
||||
|
||||
# Fused kernel
|
||||
fused_q, fused_k, fused_v, fused_g, fused_beta = fused_post_conv_prep(
|
||||
conv_output,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
num_k_heads=H,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
apply_l2norm=apply_l2norm,
|
||||
output_g_exp=output_g_exp,
|
||||
)
|
||||
|
||||
# Check shapes
|
||||
assert fused_q.shape == (L, H, K), f"q shape: {fused_q.shape}"
|
||||
assert fused_k.shape == (L, H, K), f"k shape: {fused_k.shape}"
|
||||
assert fused_v.shape == (L, HV, V), f"v shape: {fused_v.shape}"
|
||||
assert fused_g.shape == (L, HV), f"g shape: {fused_g.shape}"
|
||||
assert fused_beta.shape == (L, HV), f"beta shape: {fused_beta.shape}"
|
||||
|
||||
# Check dtypes
|
||||
assert fused_q.dtype == dtype
|
||||
assert fused_k.dtype == dtype
|
||||
assert fused_v.dtype == dtype
|
||||
assert fused_g.dtype == torch.float32
|
||||
assert fused_beta.dtype == torch.float32
|
||||
|
||||
# Check contiguity
|
||||
assert fused_q.is_contiguous()
|
||||
assert fused_k.is_contiguous()
|
||||
assert fused_v.is_contiguous()
|
||||
|
||||
# Check values
|
||||
atol_qkv = 1e-2 if apply_l2norm else 1e-3
|
||||
rtol_qkv = 1e-2 if apply_l2norm else 1e-3
|
||||
|
||||
torch.testing.assert_close(fused_q, ref_q, atol=atol_qkv, rtol=rtol_qkv)
|
||||
torch.testing.assert_close(fused_k, ref_k, atol=atol_qkv, rtol=rtol_qkv)
|
||||
torch.testing.assert_close(fused_v, ref_v, atol=1e-3, rtol=1e-3)
|
||||
torch.testing.assert_close(fused_g, ref_g, atol=1e-4, rtol=1e-4)
|
||||
torch.testing.assert_close(fused_beta, ref_beta, atol=1e-4, rtol=1e-4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("L", [1, 64, 256])
|
||||
def test_fused_post_conv_sanity(L):
|
||||
"""Sanity checks: no NaN, unit-norm q/k, beta in (0,1)."""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
H, HV, K, V = 16, 32, 128, 128
|
||||
qkv_dim = 2 * H * K + HV * V
|
||||
|
||||
conv_output = torch.randn(L, qkv_dim, dtype=torch.bfloat16, device=device)
|
||||
a = torch.randn(L, HV, dtype=torch.bfloat16, device=device)
|
||||
b = torch.randn(L, HV, dtype=torch.bfloat16, device=device)
|
||||
A_log = torch.randn(HV, dtype=torch.float32, device=device) - 2.0
|
||||
dt_bias = torch.randn(HV, dtype=torch.float32, device=device)
|
||||
|
||||
q, k, v, g, beta = fused_post_conv_prep(
|
||||
conv_output,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
num_k_heads=H,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
)
|
||||
|
||||
# Basic sanity
|
||||
assert not torch.isnan(q).any(), "NaN in q"
|
||||
assert not torch.isnan(k).any(), "NaN in k"
|
||||
assert not torch.isnan(v).any(), "NaN in v"
|
||||
assert not torch.isnan(g).any(), "NaN in g"
|
||||
assert not torch.isnan(beta).any(), "NaN in beta"
|
||||
|
||||
# L2 norm check: each head vector should have unit norm
|
||||
q_norms = torch.norm(q.float(), dim=-1)
|
||||
k_norms = torch.norm(k.float(), dim=-1)
|
||||
torch.testing.assert_close(q_norms, torch.ones_like(q_norms), atol=1e-3, rtol=1e-3)
|
||||
torch.testing.assert_close(k_norms, torch.ones_like(k_norms), atol=1e-3, rtol=1e-3)
|
||||
|
||||
# Beta should be in (0, 1)
|
||||
assert (beta >= 0).all() and (beta <= 1).all(), "beta out of range"
|
||||
|
||||
|
||||
def test_fused_post_conv_l0():
|
||||
"""Test L=0 edge case."""
|
||||
device = "cuda"
|
||||
H, HV, K, V = 16, 32, 128, 128
|
||||
qkv_dim = 2 * H * K + HV * V
|
||||
|
||||
conv_output = torch.empty(0, qkv_dim, dtype=torch.bfloat16, device=device)
|
||||
a = torch.empty(0, HV, dtype=torch.bfloat16, device=device)
|
||||
b = torch.empty(0, HV, dtype=torch.bfloat16, device=device)
|
||||
A_log = torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = torch.randn(HV, dtype=torch.float32, device=device)
|
||||
|
||||
q, k, v, g, beta = fused_post_conv_prep(
|
||||
conv_output,
|
||||
a,
|
||||
b,
|
||||
A_log,
|
||||
dt_bias,
|
||||
num_k_heads=H,
|
||||
head_k_dim=K,
|
||||
head_v_dim=V,
|
||||
)
|
||||
assert q.shape == (0, H, K)
|
||||
assert g.shape == (0, HV)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"head_ratio,tp_size,query_lengths,state_dtype,norm_dtype",
|
||||
[
|
||||
pytest.param(8, 16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
|
||||
pytest.param(8, 4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
|
||||
pytest.param(8, 16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
|
||||
pytest.param(8, 4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
|
||||
pytest.param(8, 16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
|
||||
pytest.param(8, 4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
|
||||
pytest.param(
|
||||
1,
|
||||
1,
|
||||
(4, 4),
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
id="ratio1-tp1-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
2,
|
||||
1,
|
||||
(4, 2, 0),
|
||||
torch.bfloat16,
|
||||
torch.bfloat16,
|
||||
id="ratio2-tp1-ragged-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
2,
|
||||
4,
|
||||
(8,),
|
||||
torch.float32,
|
||||
torch.float32,
|
||||
id="ratio2-tp4-max-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
3,
|
||||
2,
|
||||
(4, 2, 0),
|
||||
torch.float32,
|
||||
torch.float32,
|
||||
id="ratio3-tp2-ragged-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
4,
|
||||
4,
|
||||
(8,),
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
id="ratio4-tp4-max-fp32",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("output_gate_activation", ["silu", "sigmoid"])
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
head_ratio: int,
|
||||
tp_size: int,
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
norm_dtype: torch.dtype,
|
||||
output_gate_activation: str,
|
||||
) -> None:
|
||||
if torch.cuda.get_device_capability() < (8, 0):
|
||||
pytest.skip("fused GDN decode MTP requires compute capability 8.0+")
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
pytest.skip("fused GDN decode MTP op is not built")
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
H = 16 // tp_size
|
||||
HV = head_ratio * H
|
||||
K = V = 128
|
||||
num_reqs = len(query_lengths)
|
||||
state_width = max(query_lengths)
|
||||
num_tokens = sum(query_lengths)
|
||||
num_slots = num_reqs * state_width + 1
|
||||
scale = K**-0.5
|
||||
eps = 1e-6
|
||||
|
||||
mixed_qkv = torch.randn(
|
||||
num_tokens,
|
||||
2 * H * K + HV * V,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[H * K, H * K, HV * V],
|
||||
dim=-1,
|
||||
)
|
||||
query = query.view(1, num_tokens, H, K)
|
||||
key = key.view(1, num_tokens, H, K)
|
||||
value = value.view(1, num_tokens, HV, V)
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
b, a = ba.chunk(2, dim=-1)
|
||||
assert not a.is_contiguous()
|
||||
assert not b.is_contiguous()
|
||||
A_log = 0.5 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
dt_bias = 0.1 * torch.randn(HV, dtype=torch.float32, device=device)
|
||||
output_gate = torch.randn(num_tokens, HV, V, dtype=torch.bfloat16, device=device)
|
||||
norm_weight = torch.randn(V, dtype=norm_dtype, device=device)
|
||||
state_ref = (
|
||||
0.01 * torch.randn(num_slots, HV, V, K, dtype=torch.float32, device=device)
|
||||
).to(state_dtype)
|
||||
state_actual = state_ref.clone()
|
||||
state_indices = torch.arange(1, num_slots, dtype=torch.int32, device=device).view(
|
||||
num_reqs, state_width
|
||||
)
|
||||
cu_seqlens = torch.tensor(
|
||||
[0, *torch.tensor(query_lengths).cumsum(0).tolist()],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
num_accepted_tokens = torch.ones(num_reqs, dtype=torch.int32, device=device)
|
||||
if query_lengths[-1] == 0:
|
||||
state_indices[-1].zero_()
|
||||
|
||||
for step, accepted_tokens in enumerate((1, min(2, state_width), state_width)):
|
||||
num_accepted_tokens.fill_(accepted_tokens)
|
||||
if query_lengths[-1] == 0:
|
||||
num_accepted_tokens[-1] = 1
|
||||
raw_ref, _ = fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
initial_state=state_ref,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=cu_seqlens,
|
||||
ssm_state_indices=state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
scale=scale,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
expected = rmsnorm_fn(
|
||||
raw_ref.squeeze(0),
|
||||
norm_weight,
|
||||
None,
|
||||
z=output_gate,
|
||||
eps=eps,
|
||||
norm_before_gate=True,
|
||||
activation=output_gate_activation,
|
||||
)
|
||||
actual = ops.fused_gdn_decode_post_conv_mtp(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
state=state_actual,
|
||||
output_gate=output_gate,
|
||||
norm_weight=norm_weight,
|
||||
out=torch.empty_like(output_gate),
|
||||
scale=scale,
|
||||
norm_eps=eps,
|
||||
output_gate_activation=output_gate_activation,
|
||||
)
|
||||
|
||||
output_error = (actual.float() - expected.float()).norm()
|
||||
output_relative_l2 = output_error / expected.float().norm().clamp_min(1e-20)
|
||||
assert output_relative_l2 < 5e-4, (
|
||||
f"MTP output relative L2 mismatch at step {step}: "
|
||||
f"{output_relative_l2.item():.6g}"
|
||||
)
|
||||
|
||||
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_recurrent_gated_delta_rule,
|
||||
fused_recurrent_gated_delta_rule_packed_decode,
|
||||
)
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Gated delta rule Triton kernels require a CUDA-alike or XPU device.",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
|
||||
def test_fused_recurrent_packed_decode_matches_reference(
|
||||
dtype: torch.dtype, strided_mixed_qkv: bool
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Small but representative GDN config (Qwen3Next defaults are K=128, V=128).
|
||||
B = 32
|
||||
H = 4
|
||||
HV = 8 # grouped value attention: HV must be divisible by H
|
||||
K = 128
|
||||
V = 128
|
||||
qkv_dim = 2 * (H * K) + (HV * V)
|
||||
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
if strided_mixed_qkv:
|
||||
# Simulate a packed view into a larger projection buffer:
|
||||
# mixed_qkv.stride(0) > mixed_qkv.shape[1]
|
||||
proj = torch.randn((B, qkv_dim + 64), device=device, dtype=dtype)
|
||||
mixed_qkv = proj[:, :qkv_dim]
|
||||
else:
|
||||
mixed_qkv = torch.randn((B, qkv_dim), device=device, dtype=dtype)
|
||||
|
||||
a = torch.randn((B, HV), device=device, dtype=dtype)
|
||||
b = torch.randn((B, HV), device=device, dtype=dtype)
|
||||
A_log = torch.randn((HV,), device=device, dtype=dtype)
|
||||
dt_bias = torch.randn((HV,), device=device, dtype=dtype)
|
||||
|
||||
# Continuous batching indices (include PAD_SLOT_ID=-1 cases). Index 0 is
|
||||
# reserved as NULL_BLOCK_ID (CUDA graph padding), so valid slots start at 1.
|
||||
ssm_state_indices = torch.arange(1, B + 1, device=device, dtype=torch.int32)
|
||||
ssm_state_indices[-3:] = -1
|
||||
|
||||
state0 = torch.randn((B + 1, HV, V, K), device=device, dtype=dtype)
|
||||
state_ref = state0.clone()
|
||||
state_packed = state0.clone()
|
||||
|
||||
out_packed = torch.empty((B, 1, HV, V), device=device, dtype=dtype)
|
||||
|
||||
# Reference path: materialize contiguous Q/K/V + explicit gating.
|
||||
q, k, v = torch.split(mixed_qkv, [H * K, H * K, HV * V], dim=-1)
|
||||
q = q.view(B, H, K).unsqueeze(1).contiguous()
|
||||
k = k.view(B, H, K).unsqueeze(1).contiguous()
|
||||
v = v.view(B, HV, V).unsqueeze(1).contiguous()
|
||||
|
||||
x = a.float() + dt_bias.float()
|
||||
softplus_x = torch.where(
|
||||
x <= 20.0, torch.log1p(torch.exp(torch.clamp(x, max=20.0))), x
|
||||
)
|
||||
g = (-torch.exp(A_log.float()) * softplus_x).unsqueeze(1)
|
||||
beta = torch.sigmoid(b.float()).unsqueeze(1)
|
||||
|
||||
out_ref, state_ref = fused_recurrent_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
scale=K**-0.5,
|
||||
initial_state=state_ref,
|
||||
inplace_final_state=True,
|
||||
cu_seqlens=None,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
# Packed path: fused gating + recurrent directly from packed mixed_qkv.
|
||||
fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=A_log,
|
||||
dt_bias=dt_bias,
|
||||
scale=K**-0.5,
|
||||
initial_state=state_packed,
|
||||
out=out_packed,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
atol = 2e-2 if dtype != torch.float32 else 1e-4
|
||||
rtol = 1e-2 if dtype != torch.float32 else 1e-4
|
||||
# Output rows for PAD_SLOT_ID entries are never written (uninitialized in
|
||||
# both paths), so compare only the valid rows.
|
||||
valid = ssm_state_indices > 0
|
||||
torch.testing.assert_close(out_packed[valid], out_ref[valid], rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
def test_packed_decode_keeps_beta_in_fp32():
|
||||
device = torch.device(DEVICE)
|
||||
dtype = torch.bfloat16
|
||||
|
||||
mixed_qkv = torch.ones((1, 3), device=device, dtype=dtype)
|
||||
a = torch.zeros((1, 1), device=device, dtype=dtype)
|
||||
b = torch.full((1, 1), 0.5, device=device, dtype=dtype)
|
||||
params = torch.zeros((1,), device=device, dtype=dtype)
|
||||
ssm_state_indices = torch.ones((1,), device=device, dtype=torch.int32)
|
||||
|
||||
state_packed = torch.zeros((2, 1, 1, 1), device=device, dtype=torch.float32)
|
||||
out_packed = torch.empty((1, 1, 1, 1), device=device, dtype=dtype)
|
||||
|
||||
fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
b=b,
|
||||
A_log=params,
|
||||
dt_bias=params,
|
||||
scale=1.0,
|
||||
initial_state=state_packed,
|
||||
out=out_packed,
|
||||
ssm_state_indices=ssm_state_indices,
|
||||
)
|
||||
|
||||
expected_beta = torch.sigmoid(b.float()).squeeze()
|
||||
torch.testing.assert_close(
|
||||
state_packed[1, 0, 0, 0], expected_beta, rtol=1e-6, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def test_packed_decode_supports_large_batch_head_grid():
|
||||
B, H, HV, K, V = 1024, 8, 64, 1, 1
|
||||
device = torch.device(DEVICE)
|
||||
gates = torch.empty((B, HV), device=device)
|
||||
params = torch.empty((HV,), device=device)
|
||||
out = torch.empty((B, 1, HV, V), device=device)
|
||||
|
||||
fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv=torch.empty((B, 2 * H * K + HV * V), device=device),
|
||||
a=gates,
|
||||
b=gates,
|
||||
A_log=params,
|
||||
dt_bias=params,
|
||||
scale=1.0,
|
||||
initial_state=torch.empty((1, HV, V, K), device=device),
|
||||
out=out,
|
||||
ssm_state_indices=torch.zeros((B,), device=device, dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out).item() == 0
|
||||
@@ -0,0 +1,199 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops import (
|
||||
fused_recurrent_gated_delta_rule,
|
||||
fused_sigmoid_gating_delta_rule_update,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
@pytest.mark.parametrize("num_reqs", [1, 2, 4])
|
||||
@pytest.mark.parametrize("num_k_heads", [16])
|
||||
@pytest.mark.parametrize("num_v_heads", [32])
|
||||
@pytest.mark.parametrize("head_k_dim", [128])
|
||||
@pytest.mark.parametrize("head_v_dim", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_fused_sigmoid_gating_delta_rule_update_non_spec(
|
||||
tp_size: int,
|
||||
num_reqs: int,
|
||||
num_k_heads: int,
|
||||
num_v_heads: int,
|
||||
head_k_dim: int,
|
||||
head_v_dim: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
key_dim = head_k_dim * num_k_heads
|
||||
value_dim = head_v_dim * num_v_heads
|
||||
mixed_qkv_dim = (key_dim * 2 + value_dim) // tp_size
|
||||
seq_len = 1 # seq_len is 1 for decode
|
||||
num_tokens = num_reqs * seq_len
|
||||
total_entries = num_tokens * 2
|
||||
|
||||
mixed_qkv = torch.rand(num_tokens, mixed_qkv_dim, dtype=dtype)
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[
|
||||
key_dim // tp_size,
|
||||
key_dim // tp_size,
|
||||
value_dim // tp_size,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
query = query.view(1, num_tokens, num_k_heads, head_k_dim)
|
||||
key = key.view(1, num_tokens, num_k_heads, head_k_dim)
|
||||
value = value.view(1, num_tokens, num_v_heads, head_v_dim)
|
||||
|
||||
A_log = torch.rand(num_v_heads // tp_size, dtype=dtype)
|
||||
dt_bias = torch.rand(num_v_heads // tp_size, dtype=dtype)
|
||||
a = torch.rand(num_tokens, num_v_heads, dtype=dtype)
|
||||
b = torch.rand(num_tokens, num_v_heads, dtype=dtype)
|
||||
# Entry 0 is reserved as NULL_BLOCK_ID (CUDA graph padding), so valid
|
||||
# state indices start at 1.
|
||||
ssm_state = torch.rand(
|
||||
total_entries + 1, num_v_heads, head_k_dim, head_v_dim, dtype=dtype
|
||||
)
|
||||
state_indices = (torch.randperm(total_entries, dtype=torch.int32) + 1)[:num_tokens]
|
||||
cu_seqlens = torch.arange(0, num_tokens + 1, dtype=torch.int32)
|
||||
|
||||
beta = b.sigmoid()
|
||||
g = -A_log.float().exp() * F.softplus(a.float() + dt_bias)
|
||||
core_attn_out_ref, last_recurrent_state_ref = fused_recurrent_gated_delta_rule(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
g=g.unsqueeze(0),
|
||||
beta=beta.unsqueeze(0),
|
||||
initial_state=ssm_state.clone(),
|
||||
inplace_final_state=True,
|
||||
ssm_state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
core_attn_out, last_recurrent_state = fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
ssm_state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(core_attn_out, core_attn_out_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
last_recurrent_state, last_recurrent_state_ref, atol=1e-2, rtol=1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1])
|
||||
@pytest.mark.parametrize("num_reqs", [1, 2, 4])
|
||||
@pytest.mark.parametrize("num_k_heads", [16])
|
||||
@pytest.mark.parametrize("num_v_heads", [32])
|
||||
@pytest.mark.parametrize("head_k_dim", [128])
|
||||
@pytest.mark.parametrize("head_v_dim", [128])
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1, 3])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_fused_sigmoid_gating_delta_rule_update_spec(
|
||||
tp_size: int,
|
||||
num_reqs: int,
|
||||
num_k_heads: int,
|
||||
num_v_heads: int,
|
||||
head_k_dim: int,
|
||||
head_v_dim: int,
|
||||
num_speculative_tokens: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
key_dim = head_k_dim * num_k_heads
|
||||
value_dim = head_v_dim * num_v_heads
|
||||
mixed_qkv_dim = (key_dim * 2 + value_dim) // tp_size
|
||||
num_tokens = num_reqs * (num_speculative_tokens + 1)
|
||||
total_entries = num_tokens * 2
|
||||
|
||||
mixed_qkv = torch.rand(num_tokens, mixed_qkv_dim, dtype=dtype)
|
||||
query, key, value = torch.split(
|
||||
mixed_qkv,
|
||||
[
|
||||
key_dim // tp_size,
|
||||
key_dim // tp_size,
|
||||
value_dim // tp_size,
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
query = query.view(1, num_tokens, num_k_heads, head_k_dim)
|
||||
key = key.view(1, num_tokens, num_k_heads, head_k_dim)
|
||||
value = value.view(1, num_tokens, num_v_heads, head_v_dim)
|
||||
|
||||
A_log = torch.rand(num_v_heads // tp_size, dtype=dtype)
|
||||
dt_bias = torch.rand(num_v_heads // tp_size, dtype=dtype)
|
||||
a = torch.rand(num_tokens, num_v_heads, dtype=dtype)
|
||||
b = torch.rand(num_tokens, num_v_heads, dtype=dtype)
|
||||
# Entry 0 is reserved as NULL_BLOCK_ID (CUDA graph padding), so valid
|
||||
# state indices start at 1.
|
||||
ssm_state = torch.rand(
|
||||
total_entries + 1, num_v_heads, head_k_dim, head_v_dim, dtype=dtype
|
||||
)
|
||||
state_indices = (torch.randperm(total_entries, dtype=torch.int32) + 1)[
|
||||
:num_tokens
|
||||
].view(num_reqs, num_speculative_tokens + 1)
|
||||
num_accepted_tokens = torch.randint(
|
||||
1, num_speculative_tokens + 1, (num_reqs,), dtype=torch.int32
|
||||
)
|
||||
cu_seqlens = torch.arange(
|
||||
0, num_tokens + 1, num_speculative_tokens + 1, dtype=torch.int32
|
||||
)
|
||||
|
||||
beta = b.sigmoid()
|
||||
g = -A_log.float().exp() * F.softplus(a.float() + dt_bias)
|
||||
core_attn_out_ref, last_recurrent_state_ref = fused_recurrent_gated_delta_rule(
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
g=g.unsqueeze(0),
|
||||
beta=beta.unsqueeze(0),
|
||||
initial_state=ssm_state.clone(),
|
||||
inplace_final_state=True,
|
||||
ssm_state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
core_attn_out, last_recurrent_state = fused_sigmoid_gating_delta_rule_update(
|
||||
A_log=A_log,
|
||||
a=a,
|
||||
b=b,
|
||||
dt_bias=dt_bias,
|
||||
q=query,
|
||||
k=key,
|
||||
v=value,
|
||||
initial_state=ssm_state,
|
||||
inplace_final_state=True,
|
||||
ssm_state_indices=state_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(core_attn_out, core_attn_out_ref, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(
|
||||
last_recurrent_state, last_recurrent_state_ref, atol=1e-2, rtol=1e-2
|
||||
)
|
||||
Reference in New Issue
Block a user