[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-567
View File
@@ -1,567 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import random
from unittest.mock import patch
import pytest
import torch
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.activation import (
FastGELU,
FatreluAndMul,
GeluAndMul,
MulAndSilu,
NewGELU,
QuickGELU,
ReLUSquaredActivation,
SiluAndMul,
SiluAndMulWithClamp,
SwigluOAIAndMul,
SwigluStepAndMul,
swiglustep_and_mul_triton,
)
from vllm.model_executor.layers.fused_moe.activation import (
ApplyMoEActivationConfig,
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.half, torch.bfloat16, torch.float]
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
D = [512, 13824] # Arbitrary values for testing
SEEDS = [0]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
input = torch.empty(1, 1, 2)
output = torch.empty(1, 1, 1)
valid_token_counts = torch.ones(1, dtype=torch.int32)
with pytest.raises(NotImplementedError, match="relu2"):
apply_moe_activation(
MoEActivation.RELU2,
output,
input,
valid_token_counts=valid_token_counts,
)
def test_moe_silu_clamp_uses_native_xpu_fallback(
default_vllm_config, monkeypatch
) -> None:
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
clamp_limit = 3.0
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
output = torch.empty(2, 2)
apply_moe_activation(
MoEActivation.SILU,
output,
input,
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
)
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
input
)
torch.testing.assert_close(output, expected)
def _assert_masked_moe_activation(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
*,
dtype: torch.dtype,
mask_layout: str,
d: int,
max_num_tokens: int,
) -> None:
device = CUDA_DEVICES[0]
num_experts = 4
input_dim = 2 * d if activation.is_gated else d
if mask_layout == "flat":
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
valid_token_counts = torch.tensor(
[max_num_tokens // 2], dtype=torch.int32, device=device
)
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
else:
input = torch.randn(
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
)
valid_token_counts = torch.tensor(
[0, 1, max_num_tokens // 2, max_num_tokens],
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
)
apply_moe_activation(
activation,
output,
input,
activation_config=activation_config,
valid_token_counts=valid_token_counts,
)
batched_input = input.view(-1, max_num_tokens, input_dim)
batched_output = output.view(-1, max_num_tokens, d)
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
if num_tokens:
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
apply_moe_activation(
activation,
expected,
batched_input[expert, :num_tokens].clone(),
activation_config=activation_config,
)
torch.testing.assert_close(
batched_output[expert, :num_tokens],
expected,
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
@pytest.mark.parametrize(
"activation",
[
"silu_and_mul",
"mul_and_silu",
"gelu",
"gelu_tanh",
"fatrelu",
"swigluoai_and_mul",
"swiglustep_and_mul",
],
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("d", D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_act_and_mul(
default_vllm_config,
activation: str,
num_tokens: int,
d: int,
dtype: torch.dtype,
seed: int,
device: str,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
if activation == "silu_and_mul":
layer = SiluAndMul(compile_native=False)
fn = torch.ops._C.silu_and_mul
if activation == "mul_and_silu":
layer = MulAndSilu()
fn = torch.ops._C.mul_and_silu
elif activation == "gelu":
layer = GeluAndMul(approximate="none")
fn = torch.ops._C.gelu_and_mul
elif activation == "gelu_tanh":
layer = GeluAndMul(approximate="tanh")
fn = torch.ops._C.gelu_tanh_and_mul
elif activation == "fatrelu":
threshold = random.uniform(0, 1)
layer = FatreluAndMul(threshold)
fn = torch.ops._C.fatrelu_and_mul
elif activation == "swigluoai_and_mul":
layer = SwigluOAIAndMul()
fn = torch.ops._C.swigluoai_and_mul
elif activation == "swiglustep_and_mul":
layer = SwigluStepAndMul()
fn = swiglustep_and_mul_triton
out = layer(x)
ref_out = layer.forward_native(x)
if activation in ["swigluoai_and_mul", "swiglustep_and_mul"]:
rtol = {
# For fp16, change the relative tolerance from 1e-3 to 2e-3
torch.float16: 2e-3,
torch.bfloat16: 2e-2,
torch.float: 1.3e-6,
}
def _get_rtol(output) -> float:
return rtol[output.dtype]
torch.testing.assert_close(
out, ref_out, atol=get_default_atol(out), rtol=_get_rtol(out)
)
else:
# The SiluAndMul, MulAndSilu, GELU and FatReLU implementations are
# equivalent to the native PyTorch implementations, so we can do exact
# comparison.
torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
d = x.shape[-1] // 2
output_shape = x.shape[:-1] + (d,)
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
if activation == "fatrelu":
opcheck(fn, (out, x, threshold))
elif activation == "swigluoai_and_mul":
opcheck(fn, (out, x, layer.alpha, layer.limit))
elif activation != "swiglustep_and_mul":
opcheck(fn, (out, x))
SWIGLU_LIMITS = [3.0, 7.0, 15.0]
@torch.inference_mode()
def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None:
x = torch.randn(7, 1024, dtype=torch.bfloat16, device="cuda")
output = torch.empty(7, 512, dtype=x.dtype, device=x.device)
swiglu_limit_func(output, x, swiglu_limit=7.0)
gate, up = x.chunk(2, dim=-1)
expected = torch.nn.functional.silu(gate.clamp(max=7.0)) * up.clamp(
min=-7.0, max=7.0
)
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("alpha", "beta"),
[(1.0, 0.0), (1.702, 0.0), (1.0, 1.0)],
)
@pytest.mark.parametrize("swiglu_limit", SWIGLU_LIMITS)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("d", D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_silu_and_mul_with_clamp(
default_vllm_config,
alpha: float,
beta: float,
swiglu_limit: float,
num_tokens: int,
d: int,
dtype: torch.dtype,
seed: int,
device: str,
) -> None:
"""SiluAndMulWithClamp: cuda kernel must match native reference."""
set_random_seed(seed)
torch.set_default_device(device)
# Use large values to ensure clamping is exercised.
x = torch.randn(num_tokens, 2 * d, dtype=dtype) * swiglu_limit * 2
default_vllm_config.compilation_config.custom_ops = [
"none",
"+silu_and_mul_with_clamp",
]
layer = SiluAndMulWithClamp(
swiglu_limit,
alpha=alpha,
beta=beta,
compile_native=False,
)
if current_platform.is_rocm():
# forward_hip is always dispatched; the alpha/beta gate is checked
# inside it at call time rather than picked at construction time, so
# verify the actual routing by spying on the two candidate methods.
assert layer._forward_method == layer.forward_hip
with (
patch.object(layer, "forward_cuda", wraps=layer.forward_cuda) as cuda_spy,
patch.object(
layer, "forward_native", wraps=layer.forward_native
) as native_spy,
):
out = layer(x)
if alpha == 1.0 and beta == 0.0:
cuda_spy.assert_called_once()
native_spy.assert_not_called()
else:
native_spy.assert_called_once()
cuda_spy.assert_not_called()
else:
assert layer._forward_method == layer.forward_cuda
out = layer(x)
ref_out = layer.forward_native(x)
rtol = {
torch.float16: 2e-3,
torch.bfloat16: 2e-2,
torch.float: 1.3e-6,
}
torch.testing.assert_close(
out, ref_out, atol=get_default_atol(out), rtol=rtol[out.dtype]
)
# Verify clamping is actually being applied: the clamped output should
# differ from the unclamped SiluAndMul output when inputs are large.
if alpha == 1.0 and beta == 0.0:
unclamped_out = SiluAndMul.forward_native(x)
assert not torch.equal(ref_out.float(), unclamped_out.float()), (
"Input was not large enough to exercise the clamp; increase scale"
)
# Verify gate clamping semantics with a controlled scalar case.
# gate=large_val is clamped to limit first, then silu(limit) * 1.0.
x_gate = torch.tensor(
[[swiglu_limit * 20.0, 1.0]], dtype=torch.float32, device=device
)
out_gate = SiluAndMulWithClamp(swiglu_limit, compile_native=False)(x_gate)
expected_gate = torch.nn.functional.silu(
torch.tensor(swiglu_limit, dtype=torch.float32)
).item()
torch.testing.assert_close(
out_gate,
torch.tensor([[expected_gate]], dtype=torch.float32, device=device),
atol=1e-3,
rtol=1e-3,
)
# Verify up clamping semantics: up >> limit gets clamped to limit.
x_up = torch.tensor(
[[1.0, swiglu_limit * 20.0]], dtype=torch.float32, device=device
)
out_up = SiluAndMulWithClamp(swiglu_limit, compile_native=False)(x_up)
silu_1 = torch.nn.functional.silu(torch.tensor(1.0)).item()
torch.testing.assert_close(
out_up,
torch.tensor([[silu_1 * swiglu_limit]], dtype=torch.float32, device=device),
atol=1e-3,
rtol=1e-3,
)
# opcheck
out_buf = torch.empty(x.shape[:-1] + (d,), dtype=dtype, device=device)
opcheck(
torch.ops._C.silu_and_mul_with_clamp,
(out_buf, x, swiglu_limit, layer.alpha, layer.beta),
)
MOE_ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
pytest.param(
MoEActivation.SILU, ApplyMoEActivationConfig(clamp_limit=3.0), id="silu_clamp"
),
pytest.param(MoEActivation.GELU, ApplyMoEActivationConfig(), id="gelu"),
pytest.param(MoEActivation.GELU_TANH, ApplyMoEActivationConfig(), id="gelu_tanh"),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
id="situ",
),
pytest.param(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig(), id="swigluoai"),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
id="swigluoai_uninterleave",
),
pytest.param(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig(), id="swiglustep"),
pytest.param(
MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig(), id="silu_no_mul"
),
pytest.param(
MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig(), id="gelu_no_mul"
),
pytest.param(
MoEActivation.GELU_TANH_NO_MUL,
ApplyMoEActivationConfig(),
id="gelu_tanh_no_mul",
),
pytest.param(
MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig(), id="relu2_no_mul"
),
]
@pytest.mark.parametrize(("activation", "activation_config"), MOE_ACTIVATION_CASES)
@torch.inference_mode()
def test_masked_moe_activation_dispatch(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.bfloat16,
mask_layout="batched_experts",
d=513,
max_num_tokens=7,
)
@pytest.mark.parametrize(
("activation", "activation_config", "mask_layout"),
[
pytest.param(
MoEActivation.SILU,
ApplyMoEActivationConfig(),
"flat",
id="flat",
),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
"batched_experts",
id="batched-experts",
),
],
)
@torch.inference_mode()
def test_masked_moe_activation_grid_stride(
default_vllm_config,
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
mask_layout: str,
) -> None:
_assert_masked_moe_activation(
activation,
activation_config,
dtype=torch.half,
mask_layout=mask_layout,
d=513,
max_num_tokens=67,
)
@torch.inference_mode()
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
device = CUDA_DEVICES[0]
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
opcheck(
torch.ops._C.masked_moe_activation,
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
)
@pytest.mark.parametrize(
"activation",
[
(FastGELU, torch.ops._C.gelu_fast),
(NewGELU, torch.ops._C.gelu_new),
(QuickGELU, torch.ops._C.gelu_quick),
(ReLUSquaredActivation, torch.ops._C.relu_squared),
],
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("d", D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_activation(
default_vllm_config,
activation: type[torch.nn.Module],
num_tokens: int,
d: int,
dtype: torch.dtype,
seed: int,
device: str,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
x = torch.randn(num_tokens, d, dtype=dtype)
layer = activation[0]()
fn = activation[1]
out = layer(x)
ref_out = layer.forward_native(x)
torch.testing.assert_close(
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
)
out = torch.empty_like(x)
opcheck(fn, (out, x))
HUMMING_ACTIVATION_CASES = MOE_ACTIVATION_CASES + [
pytest.param(MoEActivation.RELU2, ApplyMoEActivationConfig(), id="relu2"),
]
HUMMING_ACTIVATION_CASES += [
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=linear_beta,
),
id=f"situ-linear-beta-{linear_beta}",
)
for linear_beta in (None, 0.0, -1.0)
]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Humming requires CUDA")
@pytest.mark.parametrize(("activation", "activation_config"), HUMMING_ACTIVATION_CASES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize(("num_tokens", "d"), [(1, 512), (7, 768), (83, 512)])
@torch.inference_mode()
def test_humming_activation_matches_framework(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
dtype: torch.dtype,
num_tokens: int,
d: int,
) -> None:
"""Compare activation math/layouts without quantization or Hadamard error."""
pytest.importorskip("humming")
from humming.ops import process_input
from vllm.model_executor.layers.quantization.utils.humming.activation import (
get_humming_activation,
)
set_random_seed(0)
width = d * 2 if activation.is_gated else d
x = 4 * torch.randn(num_tokens, width, dtype=dtype, device=CUDA_DEVICES[0])
# Exercise zero, saturation, and both sides of the clamp limits (3 and 7).
edges = x.new_tensor([0, 0.001, 1, 2.99, 3, 3.01, 6.99, 7, 7.01, 8, 16])
edges = torch.cat((-edges[1:].flip(0), edges))
x[0] = edges[torch.arange(width, device=x.device) % edges.numel()]
actual, _, _ = process_input(
x,
quant_mode="none",
hadamard_block_size=0,
**get_humming_activation(activation, activation_config),
)
expected = torch.empty(num_tokens, d, dtype=dtype, device=x.device)
if activation == MoEActivation.RELU2:
# apply_moe_activation has only the non-gated ReLU2 variant.
gate, up = x.float().chunk(2, dim=-1)
activated_gate = torch.empty_like(gate)
apply_moe_activation(MoEActivation.RELU2_NO_MUL, activated_gate, gate.clone())
expected.copy_(activated_gate * up)
else:
# The framework's non-gated ReLU2 path modifies its input in place.
apply_moe_activation(
activation, expected, x.clone(), activation_config=activation_config
)
torch.testing.assert_close(
actual,
expected,
atol=get_default_atol(expected),
rtol=get_default_rtol(expected),
)
-283
View File
@@ -1,283 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ApplyRotaryEmb CustomOp dispatch behavior.
This test ensures that RotaryEmbedding classes correctly call the appropriate
ApplyRotaryEmb methods based on the calling context:
1. RotaryEmbedding.forward_native() -> ApplyRotaryEmb.forward_native()
2. RotaryEmbedding.forward_cuda() -> ApplyRotaryEmb.forward() (auto-dispatch)
3. RotaryEmbedding.forward_hip() -> ApplyRotaryEmb.forward() (auto-dispatch)
"""
from dataclasses import dataclass
import pytest
import torch
from vllm.config import (
CompilationConfig,
VllmConfig,
get_cached_compilation_config,
set_current_vllm_config,
)
from vllm.platforms import current_platform
CUDA_DEVICES = ["cuda:0"]
@dataclass
class RotaryEmbeddingTestCase:
"""Test case configuration for RotaryEmbedding dispatch tests."""
name: str
rope_class: type
rope_kwargs: dict
method_name: str # forward_native, forward_cuda, forward
positions_shape: tuple # (num_tokens,) or (3, num_tokens) or (4, num_tokens)
expect_forward_native: bool # Should call ApplyRotaryEmb.forward_native()
expect_forward: bool # Should call ApplyRotaryEmb.forward()
def get_test_cases() -> list[RotaryEmbeddingTestCase]:
"""Generate test cases for all RotaryEmbedding classes."""
from vllm.model_executor.layers.rotary_embedding.ernie45_vl_rope import (
Ernie4_5_VLRotaryEmbedding,
)
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
common_kwargs = {
"head_size": 128,
"rotary_dim": 128,
"max_position_embeddings": 4096,
"base": 10000,
"is_neox_style": True,
"dtype": torch.bfloat16,
}
return [
# MRotaryEmbedding tests
RotaryEmbeddingTestCase(
name="MRotaryEmbedding.forward_native",
rope_class=MRotaryEmbedding,
rope_kwargs={**common_kwargs, "mrope_section": [16, 24, 24]},
method_name="forward_native",
positions_shape=(3, 32), # 2D for multimodal
expect_forward_native=True,
expect_forward=False,
),
RotaryEmbeddingTestCase(
name="MRotaryEmbedding.forward_cuda_1d",
rope_class=MRotaryEmbedding,
rope_kwargs={**common_kwargs, "mrope_section": [16, 24, 24]},
method_name="forward_cuda",
positions_shape=(32,), # 1D triggers apply_rotary_emb path
expect_forward_native=False,
expect_forward=True,
),
# Ernie4_5_VLRotaryEmbedding tests
RotaryEmbeddingTestCase(
name="Ernie4_5_VLRotaryEmbedding.forward_native",
rope_class=Ernie4_5_VLRotaryEmbedding,
rope_kwargs={**common_kwargs, "mrope_section": [22, 22, 20]},
method_name="forward_native",
positions_shape=(3, 32), # 2D for multimodal
expect_forward_native=True,
expect_forward=False,
),
]
def run_dispatch_test(
test_case: RotaryEmbeddingTestCase,
device: str,
):
"""Run a dispatch test for a RotaryEmbedding class."""
vllm_config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"])
)
get_cached_compilation_config.cache_clear()
with set_current_vllm_config(vllm_config):
rope = test_case.rope_class(**test_case.rope_kwargs).to(device=device)
apply_rotary_emb = rope.apply_rotary_emb
# Verify custom op is enabled
if test_case.expect_forward_native:
assert (
apply_rotary_emb._forward_method != apply_rotary_emb.forward_native
), "Test setup error: ApplyRotaryEmb custom op should be enabled"
# Setup call tracking
call_tracker = {"forward_native_called": False, "forward_called": False}
original_forward_native = apply_rotary_emb.forward_native
original_forward = apply_rotary_emb.forward
def tracked_forward_native(*args, **kwargs):
call_tracker["forward_native_called"] = True
return original_forward_native(*args, **kwargs)
def tracked_forward(*args, **kwargs):
call_tracker["forward_called"] = True
return original_forward(*args, **kwargs)
apply_rotary_emb.forward_native = tracked_forward_native
apply_rotary_emb.forward = tracked_forward
try:
num_tokens = test_case.positions_shape[-1]
num_q_heads = 8
num_kv_heads = 2
head_size = test_case.rope_kwargs["head_size"]
max_position = test_case.rope_kwargs["max_position_embeddings"]
positions = torch.randint(
0, max_position // 4, test_case.positions_shape, device=device
)
query = torch.randn(
num_tokens, num_q_heads * head_size, dtype=torch.bfloat16, device=device
)
key = torch.randn(
num_tokens,
num_kv_heads * head_size,
dtype=torch.bfloat16,
device=device,
)
# Call the method under test
method = getattr(rope, test_case.method_name)
method(positions, query.clone(), key.clone())
# Verify expectations
if test_case.expect_forward_native:
assert call_tracker["forward_native_called"], (
f"{test_case.name} should call ApplyRotaryEmb.forward_native()"
)
if not test_case.expect_forward:
assert not call_tracker["forward_called"], (
f"{test_case.name} should NOT call ApplyRotaryEmb.forward(). "
"Bug: when +apply_rotary_emb is enabled, forward_native() "
"incorrectly dispatches to CUDA/HIP kernels."
)
if test_case.expect_forward:
assert call_tracker["forward_called"], (
f"{test_case.name} should call ApplyRotaryEmb.forward()"
)
finally:
apply_rotary_emb.forward_native = original_forward_native
apply_rotary_emb.forward = original_forward
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
)
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda tc: tc.name)
@pytest.mark.parametrize("device", CUDA_DEVICES)
def test_rotary_embedding_dispatch(
test_case: RotaryEmbeddingTestCase,
device: str,
):
"""Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
- forward_native methods should call ApplyRotaryEmb.forward_native()
- forward_cuda/forward methods should call ApplyRotaryEmb.forward()
"""
run_dispatch_test(test_case, device)
@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU only test.")
@pytest.mark.parametrize(
"num_tokens,num_heads,head_size,rot_dim,is_neox_style",
[
(128, 32, 128, 128, True), # typical text shape
(4096, 16, 80, 80, False), # GPT-J style (interleaved)
(4096, 32, 128, 64, True), # partial rotary (rot_dim < head_size)
],
ids=["text", "gptj_interleaved", "partial_rotary"],
)
def test_apply_rotary_emb_xpu_matches_native(
num_tokens: int,
num_heads: int,
head_size: int,
rot_dim: int,
is_neox_style: bool,
):
"""ApplyRotaryEmb.forward_xpu (vllm_xpu_kernels SYCL op) must match
forward_native (the CustomOp default fallback) exactly."""
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
vllm_config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"])
)
get_cached_compilation_config.cache_clear()
with set_current_vllm_config(vllm_config):
op = ApplyRotaryEmb(enforce_enable=True, is_neox_style=is_neox_style)
x = torch.randn(
num_tokens, num_heads, head_size, device="xpu", dtype=torch.bfloat16
)
x = x[..., :rot_dim].contiguous()
cos = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
sin = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
expected = op.forward_native(x, cos, sin)
actual = op.forward_xpu(x, cos, sin)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.")
@pytest.mark.parametrize("num_tokens", [257, 2304])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_packed_qk_rope_correctness(
num_tokens: int, dtype: torch.dtype, default_vllm_config
):
"""packed_qk_rope_ must match the per-tensor ApplyRotaryEmb path with
enable_fp32_compute=True to within one ULP, and must leave the V slice
untouched.
Both sides compute in fp32 on identical inputs. The only gap is which
product the backend keeps exact inside the o1 multiply-add, and that
follows the register layout, so it is decided by codegen rather than by
the source and holds neither across backends nor across Triton versions.
"""
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
packed_qk_rope_,
)
num_heads, head_dim = 12, 128
rng = torch.Generator(device="cuda").manual_seed(0)
angles = (
torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi
)
freqs_cis = torch.polar(torch.ones_like(angles), angles)
cos = freqs_cis.real.contiguous()
sin = freqs_cis.imag.contiguous()
xqkv = torch.randn(
num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng
)
xqkv_ref = xqkv.clone()
xq, xk, xv = torch.unbind(xqkv_ref, dim=-3)
op = ApplyRotaryEmb(
enforce_enable=True, is_neox_style=False, enable_fp32_compute=True
)
# op() dispatches to forward_cuda / forward_hip per platform.
xq_ref = op(xq, cos, sin)
xk_ref = op(xk, cos, sin)
packed_qk_rope_(xqkv, freqs_cis)
# One ULP at the largest magnitude present, not relative: o1 can cancel,
# so the error scales with the inputs to the sum, not with the result.
eps = torch.finfo(dtype).eps
q_atol = eps * xq_ref.abs().max().item()
k_atol = eps * xk_ref.abs().max().item()
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=q_atol, rtol=0)
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=k_atol, rtol=0)
# V is never read or written by the kernel, so it stays exact.
torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the batched-weight RMS norm kernel (vllm._custom_ops.rms_norm).
``rms_norm`` can use the outermost input batch index to select the corresponding
weight row. The result must match that of looping ``rms_norm`` over that dimension.
"""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="rms_norm requires a CUDA/ROCm device",
)
@pytest.mark.parametrize(
"shape",
[
(28, 17, 128), # 3D: [num_rows, tokens, hidden]
(1, 5, 2, 128), # 4D: single row (edge case)
(28, 13, 8, 128), # 4D: [L, num_ctx, nkv, hd] (DFlash K-norm)
(6, 3, 4, 769), # 4D: non-power-of-two hidden size
],
)
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16, torch.float])
@pytest.mark.parametrize("seed", [42])
@torch.inference_mode()
def test_rms_norm_matches_loop(
shape: tuple[int, ...], dtype: torch.dtype, seed: int
) -> None:
set_random_seed(seed)
torch.set_default_device("cuda")
num_rows, hidden = shape[0], shape[-1]
eps = 1e-6
x = torch.randn(*shape, dtype=dtype) * 0.1
# Distinct weight per row so that a wrong row index would be caught.
weight = torch.randn(num_rows, hidden, dtype=dtype) * 0.1 + 1.0
# Reference batched-weight rms norm.
out_ref = torch.empty_like(x)
for i in range(x.shape[0]):
ops.rms_norm(out_ref[i], x[i], weight[i], eps)
out = torch.empty_like(x)
ops.rms_norm(out, x, weight, eps)
# Expect bitwise-identical results.
torch.testing.assert_close(out, out_ref, atol=0, rtol=0)
@torch.inference_mode()
def test_rms_norm_validates_shapes() -> None:
torch.set_default_device("cuda")
x = torch.randn(4, 8, 128, dtype=torch.float)
out = torch.empty_like(x)
# Expect num rows mismatch.
with pytest.raises(RuntimeError):
ops.rms_norm(out, x, torch.randn(3, 128), 1e-6)
# Expect hidden size mismatch.
with pytest.raises(RuntimeError):
ops.rms_norm(out, x, torch.randn(4, 64), 1e-6)
-212
View File
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from tests.kernels.utils import opcheck
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
from vllm.model_executor.layers.activation import (
GELU,
FastGELU,
GeluAndMul,
NewGELU,
QuickGELU,
SiluAndMul,
)
DTYPES = [torch.bfloat16, torch.float32]
NUM_TOKENS = [7, 83]
D = [512, 2048]
SEEDS = [0]
@pytest.mark.parametrize(
("activation_cls", "fn"),
[
(SiluAndMul, torch.ops._C.silu_and_mul),
(GeluAndMul, torch.ops._C.gelu_and_mul),
(GeluAndMul, torch.ops._C.gelu_tanh_and_mul),
],
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("d", D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_cpu_act_and_mul(
default_vllm_config,
activation_cls: type[torch.nn.Module],
fn: object,
num_tokens: int,
d: int,
dtype: torch.dtype,
seed: int,
) -> None:
set_random_seed(seed)
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
layer = activation_cls()
out = layer(x)
ref_out = layer.forward_native(x)
torch.testing.assert_close(
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
)
output_shape = x.shape[:-1] + (x.shape[-1] // 2,)
raw_out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
opcheck(fn, (raw_out, x))
@pytest.mark.parametrize(
("activation_cls", "fn", "op_args"),
[
(NewGELU, torch.ops._C.gelu_new, ()),
(FastGELU, torch.ops._C.gelu_fast, ()),
(QuickGELU, torch.ops._C.gelu_quick, ()),
pytest.param(
GELU,
getattr(torch.ops._C, "activation_lut_bf16", None),
("gelu",),
marks=pytest.mark.skipif(
current_platform.get_cpu_architecture() != CpuArchEnum.ARM,
reason="activation_lut_bf16 is only built on Arm CPU",
),
),
],
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("d", D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_cpu_unary_activation(
default_vllm_config,
activation_cls: type[torch.nn.Module],
fn: object,
op_args: tuple[str, ...],
num_tokens: int,
d: int,
dtype: torch.dtype,
seed: int,
) -> None:
set_random_seed(seed)
x = torch.randn(num_tokens, d, dtype=dtype)
layer = activation_cls()
out = layer(x)
ref_out = layer.forward_native(x)
torch.testing.assert_close(
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
)
# gelu with activation_lut_bf16 only makes sense for BF16
if not (activation_cls is GELU and dtype != torch.bfloat16):
raw_out = torch.empty_like(x)
opcheck(fn, (raw_out, x, *op_args))
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_cpu_gelu_tanh_and_mul(
default_vllm_config,
dtype: torch.dtype,
) -> None:
gate = torch.tensor(
[
[
-12.0,
-10.0,
-9.01,
-5.0,
-2.0,
-1.0,
-0.0,
0.0,
0.5,
1.0,
2.0,
5.0,
9.01,
10.0,
12.0,
11.0,
],
[
-7.5,
-4.5,
-3.0,
-1.5,
-0.75,
-0.25,
0.25,
0.75,
1.5,
3.0,
4.5,
7.5,
-11.0,
11.0,
8.75,
-8.75,
],
],
dtype=dtype,
)
val = torch.tensor(
[
[
0.25,
-0.5,
0.75,
-1.0,
1.25,
-1.5,
1.75,
-2.0,
2.25,
-2.5,
2.75,
-3.0,
3.25,
-3.5,
3.75,
-4.0,
],
[
-0.4,
0.6,
-0.8,
1.0,
-1.2,
1.4,
-1.6,
1.8,
-2.0,
2.2,
-2.4,
2.6,
-2.8,
3.0,
-3.2,
3.4,
],
],
dtype=dtype,
)
x = torch.cat((val, gate), dim=-1).contiguous()
kernel_out = torch.empty_like(val)
torch.ops._C.gelu_tanh_and_mul(kernel_out, x)
torch_ref = torch.nn.functional.gelu(val, approximate="tanh") * gate
atol = get_default_atol(kernel_out)
rtol = get_default_rtol(kernel_out)
torch.testing.assert_close(kernel_out, torch_ref, atol=atol, rtol=rtol)
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the manual AllReduce + GemmaRMSNorm fusion used by MiniMax M3.
``fused_allreduce_gemma_rms_norm`` must match the unfused model path, i.e.
``GemmaRMSNorm(all_reduce(partial), residual)``, both on the flashinfer fast
path (TP>1 with flashinfer + NVSwitch) and on the eager fallback (TP==1, or when
flashinfer is unavailable / the GPU has no NVSwitch).
"""
import pytest
import torch
from torch.multiprocessing import spawn
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_random_seed
@ensure_current_vllm_config()
def _worker_fused_ar_norm(
local_rank,
world_size,
port,
num_tokens,
hidden_size,
dtype,
seed,
eps,
):
"""Per-rank worker: compare the fused helper vs all_reduce + GemmaRMSNorm."""
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
world_size, 1, local_rank, port, local_rank=local_rank
)
# Norm weights are identical across ranks (replicated GemmaRMSNorm).
set_random_seed(seed)
norm = GemmaRMSNorm(hidden_size, eps=eps).cuda().to(dtype)
with torch.no_grad():
norm.weight.normal_(mean=0.0, std=0.1)
# Residual is shared across ranks; the partial o_proj output differs per rank
# (each rank holds a partial sum that all_reduce combines).
torch.manual_seed(seed + 7)
residual = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
torch.manual_seed(seed + 1000 + local_rank)
partial = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
# Reference: the unfused model path.
reduced = tensor_model_parallel_all_reduce(partial.clone())
ref_out, ref_res = norm(reduced, residual.clone())
# Fused helper (flashinfer fast path when available, else fallback).
out, res = fused_allreduce_gemma_rms_norm(partial.clone(), residual.clone(), norm)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(res, ref_res, atol=2e-2, rtol=2e-2)
cleanup_dist_env_and_memory()
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA required",
)
# world_size=1 exercises the TP==1 identity branch on a single GPU; >1 exercises
# the all_reduce + GemmaRMSNorm equivalence (flashinfer kernel or fallback).
@pytest.mark.parametrize("world_size", [1, 2, 4])
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
@pytest.mark.parametrize("hidden_size", [2048, 4096])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.parametrize("seed", [42])
def test_fused_allreduce_gemma_rms_norm(
world_size,
num_tokens,
hidden_size,
dtype,
eps,
seed,
):
num_gpus = current_platform.device_count()
if num_gpus < world_size:
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
port = str(get_open_port())
spawn(
_worker_fused_ar_norm,
args=(
world_size,
port,
num_tokens,
hidden_size,
dtype,
seed,
eps,
),
nprocs=world_size,
join=True,
)
@@ -1,148 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.bfloat16, torch.float16]
IS_NEOX = [True, False]
EPS_VALUES = [1e-5, 1e-6]
SEEDS = [13]
PARTIAL_ROPE = [True, False]
CUDA_DEVICES = ["cuda:0"]
def _apply_qk_norm_rope(
qkv: torch.Tensor,
positions: torch.Tensor,
q_norm: RMSNorm,
k_norm: RMSNorm,
rope: RotaryEmbedding,
num_heads_q: int,
num_heads_kv: int,
head_dim: int,
) -> torch.Tensor:
q_size = num_heads_q * head_dim
kv_size = num_heads_kv * head_dim
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
q_by_head = q_norm.forward_native(q_by_head)
assert isinstance(q_by_head, torch.Tensor)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
k_by_head = k_norm.forward_native(k_by_head)
assert isinstance(k_by_head, torch.Tensor)
k = k_by_head.view(k.shape)
q, k = rope.forward_native(positions, q, k)
return torch.cat([q, k, v], dim=-1)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_qk_norm_rope custom op requires cuda and rocm platform",
)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("is_neox", IS_NEOX)
@pytest.mark.parametrize("eps", EPS_VALUES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
@torch.inference_mode()
def test_fused_qk_norm_rope_matches_reference(
default_vllm_config,
device: str,
dtype: torch.dtype,
is_neox: bool,
eps: float,
seed: int,
rotary_ratio: float,
):
torch.set_default_device(device)
set_random_seed(seed)
num_heads, num_kv_heads, head_dim = 16, 4, 128
num_tokens = 4
total_dim = (num_heads + 2 * num_kv_heads) * head_dim
qkv_base = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
qkv_fused = qkv_base.clone()
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
q_norm = RMSNorm(head_dim, eps=eps).to(device=device, dtype=dtype)
k_norm = RMSNorm(head_dim, eps=eps).to(device=device, dtype=dtype)
q_norm.weight.data.normal_(mean=1.0, std=0.1)
k_norm.weight.data.normal_(mean=1.0, std=0.1)
q_weight = q_norm.weight.data
k_weight = k_norm.weight.data
rotary_dim = int(head_dim * rotary_ratio)
rope = RotaryEmbedding(
head_size=head_dim,
rotary_dim=rotary_dim,
max_position_embeddings=4096,
base=10000.0,
is_neox_style=is_neox,
dtype=dtype,
).to(device)
ref_result = _apply_qk_norm_rope(
qkv=qkv_base,
positions=positions,
q_norm=q_norm,
k_norm=k_norm,
rope=rope,
num_heads_q=num_heads,
num_heads_kv=num_kv_heads,
head_dim=head_dim,
)
opcheck(
torch.ops._C.fused_qk_norm_rope,
(
qkv_fused.clone(),
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
),
)
torch.ops._C.fused_qk_norm_rope(
qkv_fused,
num_heads,
num_kv_heads,
num_kv_heads,
head_dim,
eps,
q_weight,
k_weight,
rope.cos_sin_cache,
is_neox,
positions.view(-1),
)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(
qkv_fused,
ref_result,
atol=ATOL,
rtol=RTOL,
)
@@ -1,426 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.utils import fp8_allclose, fp8_ulp_distance, opcheck
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_group_quant_int8,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
ON_GFX950 = False
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
ON_GFX950 = on_gfx950()
DTYPES = [torch.bfloat16, torch.float]
QUANT_DTYPES = [torch.int8, current_platform.fp8_dtype()]
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
# Avoid combinatorial explosion with full Cartesian product
NUM_TOKENS_HIDDEN_SIZES = [
*[(1, i) for i in [1, 64, 128, *VEC_HIDDEN_SIZES, 5120, 5137]],
*[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
*[(4096, i) for i in [1, 64, 5137]],
]
ADD_RESIDUAL = [False, True]
SCALE_UBS = [True, False]
GROUP_SIZES = [[1, 64], [1, 128]]
TMA_ALIGNMENTS = [0, 4]
SEEDS = [0]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
EPS = 1e-6
def _is_valid_config(
hidden_size: int,
has_scale_ub: bool,
quant_dtype: torch.dtype,
group_size: list[int] | None,
tma_alignment: int,
) -> bool:
if group_size is not None and hidden_size % group_size[1] != 0:
return False
if group_size is not None and has_scale_ub:
return False
if (
group_size is None or quant_dtype != current_platform.fp8_dtype()
) and tma_alignment != 0:
return False
if (
group_size is not None
and tma_alignment != 0
and hidden_size // group_size[1] % tma_alignment == 0
):
return False
return not (has_scale_ub and quant_dtype != current_platform.fp8_dtype())
def _config_id(
num_tokens: int,
hidden_size: int,
has_scale_ub: bool,
quant_dtype: torch.dtype,
group_size: list[int] | None,
tma_alignment: int,
) -> str:
quant = str(quant_dtype).removeprefix("torch.")
group = "per-token" if group_size is None else f"group-{group_size[1]}"
return (
f"{num_tokens}x{hidden_size}-{quant}-{group}-"
f"tma-{tma_alignment}-scale-ub-{has_scale_ub}"
)
# Filter unsupported combinations during collection. Letting each case call
# pytest.skip still runs the global per-test teardown and dominates this suite.
RMS_NORM_CONFIGS = [
pytest.param(
num_tokens,
hidden_size,
has_scale_ub,
quant_dtype,
group_size,
tma_alignment,
id=_config_id(
num_tokens,
hidden_size,
has_scale_ub,
quant_dtype,
group_size,
tma_alignment,
),
)
for (
(num_tokens, hidden_size),
has_scale_ub,
quant_dtype,
(group_size, tma_alignment),
) in itertools.product(
NUM_TOKENS_HIDDEN_SIZES,
SCALE_UBS,
QUANT_DTYPES,
[(None, 0), *itertools.product(GROUP_SIZES, TMA_ALIGNMENTS)],
)
if _is_valid_config(
hidden_size,
has_scale_ub,
quant_dtype,
group_size,
tma_alignment,
)
]
## Helpers
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
return torch.as_tensor(x, dtype=torch.float32, device="cuda")
def ref_rms_norm(
rms_norm_layer: RMSNorm, x: torch.Tensor, residual: torch.Tensor | None
) -> tuple[torch.Tensor, torch.Tensor | None]:
if residual is not None:
residual = residual.clone()
out, residual = rms_norm_layer.forward_native(x, residual)
else:
out = rms_norm_layer.forward_native(x)
return out, residual
def ref_dynamic_per_token_or_block_quant(
rms_norm_layer: RMSNorm,
x: torch.Tensor,
quant_dtype: torch.dtype,
residual: torch.Tensor | None,
scale_ub: torch.Tensor | None,
group_size: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
if scale_ub is not None:
assert quant_dtype == current_platform.fp8_dtype()
# Norm
torch_out, residual = ref_rms_norm(rms_norm_layer, x, residual)
# Quant
if group_size is not None:
if quant_dtype == current_platform.fp8_dtype():
torch_out, scales = per_token_group_quant_fp8(
torch_out, group_size=group_size[1], use_ue8m0=False
)
else:
assert quant_dtype == torch.int8
torch_out, scales = per_token_group_quant_int8(
torch_out, group_size=group_size[1]
)
else:
if quant_dtype == current_platform.fp8_dtype():
torch_out, scales = ops.scaled_fp8_quant(
torch_out, scale_ub=scale_ub, use_per_token_if_dynamic=True
)
else:
assert quant_dtype == torch.int8
torch_out, scales, _ = ops.scaled_int8_quant(torch_out)
return torch_out, scales, residual
def ref_impl(
rms_norm_layer: RMSNorm,
x: torch.Tensor,
quant_dtype: torch.dtype,
residual: torch.Tensor | None,
scale_ub: torch.Tensor | None,
group_size: list[int] | None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
return ref_dynamic_per_token_or_block_quant(
rms_norm_layer, x, quant_dtype, residual, scale_ub, group_size
)
def ops_dynamic_per_token_or_block_quant(
weight: torch.Tensor,
x: torch.Tensor,
quant_dtype: torch.dtype,
residual: torch.Tensor | None,
scale_ub: torch.Tensor | None,
group_size: list[int] | None,
tma_alignment: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
if residual is not None:
residual = residual.clone()
if group_size is not None:
out, scales = ops.rms_norm_per_block_quant(
x,
weight,
EPS,
quant_dtype,
group_size,
scale_ub,
residual,
True,
tma_alignment,
)
scales = scales.contiguous()
else:
out, scales = ops.rms_norm_dynamic_per_token_quant(
x, weight, EPS, quant_dtype, scale_ub, residual
)
return out, scales, residual
def ops_impl(
weight: torch.Tensor,
x: torch.Tensor,
quant_dtype: torch.dtype,
residual: torch.Tensor | None,
scale_ub: torch.Tensor | None,
group_size: list[int] | None,
tma_alignment: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
return ops_dynamic_per_token_or_block_quant(
weight, x, quant_dtype, residual, scale_ub, group_size, tma_alignment
)
@pytest.mark.parametrize(
(
"num_tokens",
"hidden_size",
"has_scale_ub",
"quant_dtype",
"group_size",
"tma_alignment",
),
RMS_NORM_CONFIGS,
)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("strided_input", [False, True])
@torch.inference_mode()
def test_rms_norm(
default_vllm_config,
num_tokens: int,
hidden_size: int,
add_residual: bool,
has_scale_ub: bool,
dtype: torch.dtype,
quant_dtype: torch.dtype,
group_size: list[int] | None,
tma_alignment: int,
seed: int,
device: str,
strided_input: bool,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
torch.accelerator.set_device_index(device)
layer = RMSNorm(hidden_size, EPS).to(dtype=dtype)
# Make weights
layer.weight.data.normal_(mean=1.0, std=0.1)
# Make inputs: use a wider tensor and slice to create a non-contiguous
# (strided) input when strided_input=True. The last dimension stride
# remains 1, which the kernel requires.
scale = 1 / (hidden_size)
last_dim = 2 * hidden_size if strided_input else hidden_size
x = torch.randn(num_tokens, last_dim, dtype=dtype) * scale
x = x[:, :hidden_size]
# dim 1 gets special-cased
x_is_strided = strided_input and num_tokens != 1
# check that the input is strided iff we expect it to be
assert x.is_contiguous() != x_is_strided
# Residual must still be contiguous
residual = (
torch.randn(num_tokens, hidden_size, dtype=dtype) * scale
if add_residual
else None
)
if has_scale_ub:
rms_x, _ = ref_rms_norm(layer, x, residual)
scale_ub = torch.mean(rms_x).to(dtype=torch.float32, device="cuda")
else:
scale_ub = None
ref_out, ref_scales, ref_residual = ref_impl(
layer, x, quant_dtype, residual, scale_ub, group_size
)
ops_out, ops_scales, ops_residual = ops_impl(
layer.weight, x, quant_dtype, residual, scale_ub, group_size, tma_alignment
)
assert ref_out.dtype == quant_dtype
assert ops_out.dtype == quant_dtype
# Per-block bf16 scales: allow a small relative tolerance for a few groups
# whose abs-max flips by one ULP between the fused and reference paths. The
# per-token and fp32 paths stay strict.
# The same one-ULP group-scale flip also occurs on CUDA (H200), so extend
# the block relaxation there too — the fused groupwise reduction rounds
# differently than the reference for isolated groups.
relax_block = (
group_size is not None
and dtype == torch.bfloat16
and (current_platform.is_rocm() or current_platform.is_cuda())
)
use_gfx950_fp8_allclose = (
current_platform.is_rocm()
and ON_GFX950
and group_size is None
and dtype == torch.bfloat16
and quant_dtype == current_platform.fp8_dtype()
)
allow_cuda_fp8_rounding_outliers = (
current_platform.is_cuda()
and group_size is None
and dtype == torch.bfloat16
and quant_dtype == current_platform.fp8_dtype()
)
def scales_close(rtol: float, atol: float) -> bool:
if torch.allclose(ref_scales, ops_scales, rtol=rtol, atol=atol):
return True
return relax_block and torch.allclose(
ref_scales, ops_scales, rtol=1e-2, atol=atol
)
if quant_dtype == torch.int8:
assert scales_close(rtol=1e-5, atol=1e-6)
# big atol to account for round-off errors.
assert torch.allclose(ref_out, ops_out, atol=1)
else:
assert scales_close(rtol=1e-5, atol=1e-8)
a = ref_out.to(dtype=torch.float32)
b = ops_out.to(dtype=torch.float32)
ok = torch.allclose(a, b, atol=1e-6)
if not ok:
if relax_block:
# ULP-flipped group scale can cross an E4M3 tie; tolerate a
# bounded count of isolated fp8 outliers.
ulp = fp8_ulp_distance(ref_out, ops_out)
max_outliers = ulp.numel() // 100_000 + 8
ok = int((ulp > 0).sum().item()) <= max_outliers
elif use_gfx950_fp8_allclose:
# Valid gfx950 reduction trees can straddle an E4M3 boundary.
ok = fp8_allclose(ops_out, ref_out, rtol=0.125, atol=2e-3)
ok = ok and int(fp8_ulp_distance(ops_out, ref_out).max()) <= 1
elif allow_cuda_fp8_rounding_outliers:
# A valid BF16 reduction can cross an E4M3 boundary for
# isolated values.
ulp = fp8_ulp_distance(ref_out, ops_out)
max_outliers = ulp.numel() // 100_000 + 8
ok = int(ulp.max()) <= 1
ok = ok and int((ulp > 0).sum().item()) <= max_outliers
else:
# Compare dequantized values with relaxed tolerance.
if group_size is None:
a_deq = a * ref_scales.view(-1, 1)
b_deq = b * ops_scales.view(-1, 1)
else:
a_deq = a * ref_scales.repeat_interleave(group_size[1], dim=1)
b_deq = b * ops_scales.repeat_interleave(group_size[1], dim=1)
# NOTE: It is possible that some future test cases trigger this
# max diff due to precision issues. If such an error is
# encountered, it's recommended to inspect the differences between
# all corresponding elements from each tensor (e.g. by looping over
# them) and checking how many the max diff error shows up on (just
# a few bad elements should still be considered acceptable).
ok = torch.allclose(a_deq, b_deq, rtol=5e-2, atol=5e-2)
assert ok
if add_residual:
assert torch.allclose(ref_residual, ops_residual)
output = torch.empty(x.shape, dtype=quant_dtype, device=x.device)
if group_size is None:
scales = torch.empty(
(x.numel() // x.shape[-1], 1), device=x.device, dtype=torch.float32
)
opcheck(
torch.ops._C.rms_norm_dynamic_per_token_quant,
(output, x, layer.weight, scales, 1e-5, scale_ub, residual),
)
else:
assert hidden_size % group_size[1] == 0
num_groups = hidden_size // group_size[1]
scales = torch.empty(
(num_groups, num_tokens),
device=x.device,
dtype=torch.float32,
).transpose(0, 1)
opcheck(
torch.ops._C.rms_norm_per_block_quant,
(
output,
x,
layer.weight,
scales,
1e-5,
scale_ub,
residual,
group_size[1],
True, # is_scale_transposed
),
)
@@ -1,115 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
matching the eager triton kernel output."""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
FusedRMSNormGated,
)
from vllm.utils.torch_utils import set_random_seed
DEVICE = "xpu:0" if current_platform.is_xpu() else "cuda:0"
DTYPES = [torch.bfloat16]
HIDDEN_SIZES = [128, 512]
NUM_TOKENS = [64, 128]
ACTIVATIONS = ["swish", "sigmoid"]
ELEMENTWISE_AFFINE = [True, False]
SEEDS = [0]
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("activation", ACTIVATIONS)
@pytest.mark.parametrize("elementwise_affine", ELEMENTWISE_AFFINE)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_compiled_vs_eager(
default_vllm_config,
num_tokens: int,
hidden_size: int,
activation: str,
elementwise_affine: bool,
dtype: torch.dtype,
seed: int,
) -> None:
"""forward_native decomposition matches forward_cuda triton kernel."""
torch._dynamo.reset()
set_random_seed(seed)
device = torch.device(DEVICE)
module = FusedRMSNormGated(
hidden_size,
elementwise_affine=elementwise_affine,
eps=1e-5,
activation=activation,
device=device,
dtype=dtype,
)
# Model parameters use torch.empty because checkpoint loading overwrites
# them. Initialize the standalone test module so allocator contents cannot
# introduce NaNs and make this comparison flaky.
if module.weight is not None:
module.weight.uniform_(-1, 1)
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
g = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
# forward_cuda may modify x in-place, so clone inputs
cuda_out = module.forward_cuda(x.clone(), g.clone())
compiled_native = torch.compile(module.forward_native, fullgraph=True)
native_out = compiled_native(x.clone(), g.clone())
torch.testing.assert_close(native_out, cuda_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize(
"shape",
[
(1, 16, 32, 128),
(2, 8, 16, 64),
],
)
@pytest.mark.parametrize("activation", ACTIVATIONS)
@pytest.mark.parametrize("elementwise_affine", ELEMENTWISE_AFFINE)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_compiled_vs_eager_multidim(
default_vllm_config,
shape: tuple,
activation: str,
elementwise_affine: bool,
dtype: torch.dtype,
seed: int,
) -> None:
"""forward_native decomposition handles multi-dimensional inputs."""
torch._dynamo.reset()
set_random_seed(seed)
device = torch.device(DEVICE)
head_dim = shape[-1]
module = FusedRMSNormGated(
head_dim,
elementwise_affine=elementwise_affine,
eps=1e-5,
activation=activation,
device=device,
dtype=dtype,
)
if module.weight is not None:
module.weight.uniform_(-1, 1)
x = torch.randn(*shape, dtype=dtype, device=device)
g = torch.randn(*shape, dtype=dtype, device=device)
# forward_cuda may modify x in-place, so clone inputs
cuda_out = module.forward_cuda(x.clone(), g.clone())
compiled_native = torch.compile(module.forward_native, fullgraph=True)
native_out = compiled_native(x.clone(), g.clone())
torch.testing.assert_close(native_out, cuda_out, atol=1e-3, rtol=1e-2)
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
import vllm._custom_ops as ops
from tests.kernels.utils import opcheck
from tests.utils import large_gpu_test
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_group_quant_int8,
)
from vllm.platforms import current_platform
DTYPES = [torch.float16, torch.bfloat16]
QUANT_DTYPES = [current_platform.fp8_dtype(), torch.int8]
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
NUM_TOKENS_HIDDEN_SIZES = [
*[(1, i) for i in [64, *VEC_HIDDEN_SIZES, 2048, 5120]],
*[(16, i) for i in [64, *VEC_HIDDEN_SIZES, 5120]],
*[(128, i) for i in [64, *VEC_HIDDEN_SIZES]],
*[(512, i) for i in [64, 5120]],
]
SCALE_UBS = [False]
GROUP_SIZES = [64, 128]
IS_SCALE_TRANSPOSED = [False, True]
SEEDS = [0]
CUDA_DEVICES = [i for i in range(1 if torch.accelerator.device_count() == 1 else 2)]
def ref_silu_and_mul_per_block_quant(
x: torch.Tensor,
quant_dtype: torch.dtype,
group_size: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Reference implementation: unfused SiLU+Mul then group quantization."""
hidden = x.shape[-1] // 2
gate, up = x.split(hidden, dim=-1)
silu_out = F.silu(gate) * up
if quant_dtype == current_platform.fp8_dtype():
return per_token_group_quant_fp8(
silu_out, group_size=group_size, use_ue8m0=False
)
elif quant_dtype == torch.int8:
return per_token_group_quant_int8(silu_out, group_size=group_size)
else:
raise ValueError(f"Unsupported quant_dtype: {quant_dtype}")
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
@pytest.mark.parametrize("group_size", GROUP_SIZES)
@pytest.mark.parametrize("is_scale_transposed", IS_SCALE_TRANSPOSED)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device_idx", CUDA_DEVICES)
@torch.inference_mode()
def test_silu_and_mul_per_block_quant(
default_vllm_config,
num_tokens: int,
hidden_size: int,
has_scale_ub: bool,
dtype: torch.dtype,
quant_dtype: torch.dtype,
group_size: int,
is_scale_transposed: bool,
seed: int,
device_idx: str,
) -> None:
"""Test SiLU+Mul+Block Quantization kernel correctness."""
torch.accelerator.set_device_index(device_idx)
device = f"cuda:{device_idx}"
torch.random.manual_seed(seed)
torch.set_default_device(device)
if hidden_size % group_size != 0:
return
if has_scale_ub:
pytest.skip("Scale upper bound not yet supported")
scale = 1 / hidden_size
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device) * scale
# Reference implementation
ref_out, ref_scales = ref_silu_and_mul_per_block_quant(x, quant_dtype, group_size)
# Fused kernel implementation
ops_out, ops_scales = ops.silu_and_mul_per_block_quant(
x, group_size, quant_dtype, None, is_scale_transposed
)
# Check for NaN/Inf
assert not torch.isnan(ops_out.float()).any(), "Kernel output contains NaN"
assert not torch.isinf(ops_out.float()).any(), "Kernel output contains Inf"
assert not torch.isnan(ops_scales).any(), "Kernel scales contain NaN"
assert not torch.isinf(ops_scales).any(), "Kernel scales contain Inf"
# Check dtypes
assert ref_out.dtype == quant_dtype
assert ops_out.dtype == quant_dtype
# Check scales match
torch.testing.assert_close(ref_scales, ops_scales, rtol=1e-5, atol=1e-5)
# Check output correctness via dequantized values
ref_scales_expanded = ref_scales.repeat_interleave(group_size, dim=1)
ops_scales_expanded = ops_scales.repeat_interleave(group_size, dim=1)
ref_deq = ref_out.to(dtype=torch.float32) * ref_scales_expanded
ops_deq = ops_out.to(dtype=torch.float32) * ops_scales_expanded
torch.testing.assert_close(ref_deq, ops_deq, atol=5e-2, rtol=5e-2)
# opcheck
output = torch.empty(num_tokens, hidden_size, device=device, dtype=quant_dtype)
num_groups = hidden_size // group_size
if is_scale_transposed:
scales = torch.empty(num_groups, num_tokens, device=device, dtype=torch.float32)
else:
scales = torch.empty(num_tokens, num_groups, device=device, dtype=torch.float32)
opcheck(
torch.ops._C.silu_and_mul_per_block_quant,
(output, x, scales, group_size, None, is_scale_transposed),
)
@pytest.mark.parametrize("dtype", [torch.float16])
@pytest.mark.parametrize("hidden_size", [4096])
@pytest.mark.parametrize("num_tokens", [128])
@pytest.mark.parametrize("group_size", [128])
def test_silu_block_quant_shapes(
default_vllm_config,
dtype: torch.dtype,
hidden_size: int,
num_tokens: int,
group_size: int,
):
"""Test that output shapes are correct."""
torch.set_default_device("cuda")
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device="cuda")
# Row-major scales
out, scales = ops.silu_and_mul_per_block_quant(
x,
group_size=group_size,
quant_dtype=current_platform.fp8_dtype(),
is_scale_transposed=False,
)
assert out.shape == (num_tokens, hidden_size)
assert scales.shape == (num_tokens, hidden_size // group_size)
# Column-major scales (logical shape same after .t() in _custom_ops)
out, scales = ops.silu_and_mul_per_block_quant(
x,
group_size=group_size,
quant_dtype=current_platform.fp8_dtype(),
is_scale_transposed=True,
)
assert out.shape == (num_tokens, hidden_size)
assert scales.shape == (num_tokens, hidden_size // group_size)
@pytest.mark.parametrize("dtype", [torch.float16])
@pytest.mark.parametrize("batch_size", [1, 16, 256])
@pytest.mark.parametrize("hidden_size", [1024, 5120, 14336])
def test_silu_block_quant_edge_cases(
default_vllm_config, dtype: torch.dtype, batch_size: int, hidden_size: int
):
"""Test edge cases: single token, large batch, large hidden size."""
torch.set_default_device("cuda")
x = torch.randn(batch_size, hidden_size * 2, dtype=dtype, device="cuda")
out, scales = ops.silu_and_mul_per_block_quant(
x,
group_size=128,
quant_dtype=current_platform.fp8_dtype(),
is_scale_transposed=False,
)
assert out.shape == (batch_size, hidden_size)
assert out.dtype == current_platform.fp8_dtype()
assert scales.dtype == torch.float32
assert not torch.isnan(out.float()).any()
assert not torch.isnan(scales).any()
assert not torch.isinf(scales).any()
@large_gpu_test(min_gb=24)
@torch.inference_mode()
def test_silu_block_quant_int32_offset_overflow(default_vllm_config):
"""Token offsets beyond INT32_MAX must not wrap (issue #53390)."""
hidden_size = 16384
group_size = 128
quant_dtype = current_platform.fp8_dtype()
# Smallest batch whose last token's input offset (token_idx * 2 *
# hidden_size) exceeds INT32_MAX.
num_tokens = (2**31 - 1) // (2 * hidden_size) + 2
x = torch.randn(num_tokens, hidden_size * 2, dtype=torch.bfloat16, device="cuda")
out, scales = ops.silu_and_mul_per_block_quant(
x, group_size, quant_dtype, None, False
)
torch.accelerator.synchronize()
expected_out, expected_scales = ops.silu_and_mul_per_block_quant(
x[-1:].contiguous(), group_size, quant_dtype, None, False
)
assert torch.equal(scales[-1:], expected_scales)
assert torch.equal(out[-1:], expected_out)
-260
View File
@@ -1,260 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.quant_utils import FP8_DTYPE
from tests.kernels.utils import fp8_allclose, fp8_ulp_distance, opcheck
from vllm import ir
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx90a
on_mi250 = on_gfx90a()
else:
on_mi250 = False
DTYPES = [torch.half, torch.bfloat16, torch.float]
NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
HIDDEN_SIZES = [8, 768, 769, 5120, 5125, 8192] # Arbitrary values for testing
ADD_RESIDUAL = [False, True] if not on_mi250 else [True]
SEEDS = [0]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
def _rms_norm_tolerance(dtype: torch.dtype) -> dict[str, float]:
return ir.ops.rms_norm.get_tolerance(dtype)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("strided_input", [False, True])
@torch.inference_mode()
def test_rms_norm(
default_vllm_config,
num_tokens: int,
hidden_size: int,
add_residual: bool,
dtype: torch.dtype,
seed: int,
device: str,
strided_input: bool,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
layer = RMSNorm(hidden_size).to(dtype=dtype)
layer.weight.data.normal_(mean=1.0, std=0.1)
scale = 1 / (2 * hidden_size)
last_dim = 2 * hidden_size if strided_input else hidden_size
x = torch.randn(num_tokens, last_dim, dtype=dtype)
x = x[..., :hidden_size]
assert x.is_contiguous() != strided_input
x *= scale
residual = x.new_empty_strided(x.size(), x.stride()) if add_residual else None
if residual is not None:
residual.normal_(std=scale)
# NOTE(woosuk): The reference implementation should be executed first
# because the custom kernel is in-place.
ref_out = layer.forward_native(x, residual)
out = layer(x, residual)
# NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
# numerical errors than other operators because they involve reductions.
# Therefore, we use a larger tolerance.
if add_residual:
torch.testing.assert_close(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
else:
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
if residual is not None:
opcheck(
torch.ops._C.fused_add_rms_norm,
(x, residual, layer.weight.data, layer.variance_epsilon),
)
else:
opcheck(
torch.ops._C.rms_norm, (out, x, layer.weight.data, layer.variance_epsilon)
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_rms_norm_weightless(
default_vllm_config,
num_tokens: int,
hidden_size: int,
add_residual: bool,
dtype: torch.dtype,
seed: int,
device: str,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
layer = RMSNorm(hidden_size, has_weight=False).to(dtype=dtype)
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
residual = torch.randn_like(x) if add_residual else None
ref_out = layer.forward_native(x, residual)
out = layer(x, residual)
tol = _rms_norm_tolerance(dtype)
if add_residual:
torch.testing.assert_close(out[0], ref_out[0], **tol)
torch.testing.assert_close(out[1], ref_out[1], **tol)
else:
torch.testing.assert_close(out, ref_out, **tol)
if residual is not None:
opcheck(
torch.ops._C.fused_add_rms_norm,
(x, residual, None, layer.variance_epsilon),
)
else:
opcheck(
torch.ops._C.rms_norm,
(out, x, None, layer.variance_epsilon),
)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("quant_scale", [0.01, 1.0, 10.0])
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("strided_input", [False, True])
def test_fused_rms_norm_quant(
num_tokens: int,
hidden_size: int,
add_residual: bool,
dtype: torch.dtype,
quant_scale: float,
seed: int,
device: str,
strided_input: bool,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
weight = torch.empty(hidden_size, dtype=dtype).normal_(mean=1.0, std=0.1)
scale = 1 / (2 * hidden_size)
last_dim = 2 * hidden_size if strided_input else hidden_size
x_base = torch.randn(num_tokens, last_dim, dtype=dtype)
x = x_base[..., :hidden_size]
assert x.is_contiguous() != strided_input
x *= scale
if add_residual:
residual = torch.randn_like(x) * scale
residual_fused = residual.clone()
else:
residual = residual_fused = None
out_norm = torch.empty_like(x)
out_quant = torch.empty_like(x, dtype=FP8_DTYPE)
out_quant_fused = torch.empty_like(out_quant)
quant_scale_t = torch.tensor(quant_scale, dtype=torch.float32)
if add_residual:
torch.ops._C.fused_add_rms_norm_static_fp8_quant(
out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6
)
# Unfused kernel is in-place so it goes second
# Also use a separate clone of x to avoid modifying the input
x_unfused_base = x_base.clone()
x_unfused = x_unfused_base[..., :hidden_size]
assert x_unfused.is_contiguous() != strided_input
torch.ops._C.fused_add_rms_norm(x_unfused, residual, weight, 1e-6)
torch.ops._C.static_scaled_fp8_quant(
out_quant, x_unfused.contiguous(), quant_scale_t
)
torch.accelerator.synchronize()
torch.testing.assert_close(residual_fused, residual, atol=1e-2, rtol=1e-2)
opcheck(
torch.ops._C.fused_add_rms_norm_static_fp8_quant,
(out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6),
)
else:
torch.ops._C.rms_norm_static_fp8_quant(
out_quant_fused, x, weight, quant_scale_t, 1e-6
)
torch.ops._C.rms_norm(out_norm, x, weight, 1e-6)
torch.ops._C.static_scaled_fp8_quant(out_quant, out_norm, quant_scale_t)
opcheck(
torch.ops._C.rms_norm_static_fp8_quant,
(out_quant_fused, x, weight, quant_scale_t, 1e-6),
)
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
if on_gfx950() and dtype == torch.float16 and not add_residual:
# Fusion may round normalized FP16 across an E4M3 boundary on gfx950.
assert fp8_allclose(out_quant_fused, out_quant, rtol=0.125, atol=2e-3)
assert int(fp8_ulp_distance(out_quant_fused, out_quant).max()) <= 1
else:
# Fused and unfused FP8 paths can land on opposite sides of an E4M3
# tie; tolerate a tiny number of isolated fp8 outliers on ROCm.
ulp = fp8_ulp_distance(out_quant, out_quant_fused)
max_outliers = ulp.numel() // 100_000 + 8
num_outliers = int((ulp > 0).sum().item())
assert num_outliers <= max_outliers, (
f"FP8 quant mismatch: {num_outliers} fp8 outliers "
f"(allowed {max_outliers})"
)
else:
torch.testing.assert_close(
out_quant.to(dtype=torch.float32),
out_quant_fused.to(dtype=torch.float32),
atol=1e-3,
rtol=1e-3,
)
@torch.inference_mode()
def test_gemma_rms_norm_mixed_input_weight_dtype(default_vllm_config) -> None:
if not torch.cuda.is_available():
pytest.skip("CUDA required")
device = CUDA_DEVICES[0]
torch.set_default_device(device)
num_tokens, hidden_size = 32, 1024
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
layer = GemmaRMSNorm(hidden_size, eps=1e-6).to(device=device)
layer.weight.data.normal_(mean=0.0, std=0.1)
# Gemma uses fp32 weight parameter while activations can be bf16.
assert layer.weight.dtype == torch.float32
out = layer(x)
x_fp32 = x.float()
weight_fp32 = layer.weight.data.float() + 1.0
variance = x_fp32.pow(2).mean(dim=-1, keepdim=True)
ref = (x_fp32 * torch.rsqrt(variance + layer.variance_epsilon) * weight_fp32).to(
x.dtype
)
assert out.dtype == x.dtype
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@@ -1,320 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MiniMax QK RMS-norm fused all-reduce kernels and fallbacks."""
from unittest.mock import patch
import pytest
import torch
import torch.nn as nn
from torch.multiprocessing import spawn
from tests.kernels.utils import opcheck
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.minimax_rms_norm import (
MiniMaxText01RMSNormTP,
rms_norm_tp,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_random_seed
@ensure_current_vllm_config()
def _worker_forward_qk(
local_rank,
world_size,
port,
num_tokens,
hidden_q_full,
hidden_k_full,
dtype,
seed,
eps,
):
"""Per-rank worker: compare NCCL allreduce path vs Lamport fused kernel."""
if not hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
cleanup_dist_env_and_memory()
return
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
world_size, 1, local_rank, port, local_rank=local_rank
)
hq = hidden_q_full // world_size
hk = hidden_k_full // world_size
q_norm = MiniMaxText01RMSNormTP(hidden_q_full, eps=eps).cuda()
k_norm = MiniMaxText01RMSNormTP(hidden_k_full, eps=eps).cuda()
set_random_seed(seed)
qw = torch.randn(hidden_q_full, dtype=dtype, device="cuda")
kw = torch.randn(hidden_k_full, dtype=dtype, device="cuda")
q_norm.weight = nn.Parameter(qw[local_rank * hq : (local_rank + 1) * hq])
k_norm.weight = nn.Parameter(kw[local_rank * hk : (local_rank + 1) * hk])
torch.manual_seed(seed + 1000 + local_rank)
qkv = torch.randn(num_tokens, hq + hk + hk, dtype=dtype, device="cuda")
# Reference: eager all-reduce path. ``forward_qk`` no longer all-reduces
# the variance (it is the tp==1 / already-reduced building block), so the
# multi-rank reference must use the eager path that performs the global
# variance all-reduce, matching the fused kernel below.
ref_q, ref_k = rms_norm_tp._minimax_qk_norm_tp_eager(
qkv.clone(),
q_norm.weight,
k_norm.weight,
hq,
hk,
world_size,
eps,
)
# Set up Lamport workspace.
from vllm.distributed.parallel_state import get_tp_group
from vllm.model_executor.layers.minimax_rms_norm.lamport_workspace import (
get_allreduce_workspace,
)
workspace = get_allreduce_workspace(
rank=local_rank,
world_size=world_size,
max_tokens=num_tokens,
process_group=get_tp_group().cpu_group,
)
opcheck(
torch.ops._C.minimax_allreduce_rms_qk,
(
qkv.clone(),
q_norm.weight,
k_norm.weight,
workspace,
hq,
hk,
local_rank,
world_size,
eps,
),
)
fused_q, fused_k = torch.ops._C.minimax_allreduce_rms_qk(
qkv.clone(),
q_norm.weight,
k_norm.weight,
workspace,
hq,
hk,
local_rank,
world_size,
eps,
)
_, _, fused_v = qkv.split([hq, hk, hk], dim=-1)
torch.accelerator.synchronize()
torch.testing.assert_close(
fused_q,
ref_q,
atol=3e-2,
rtol=3e-2,
)
torch.testing.assert_close(fused_k, ref_k, atol=3e-2, rtol=3e-2)
cleanup_dist_env_and_memory()
@ensure_current_vllm_config()
def _worker_forward_qk_rocm_aiter(
local_rank: int,
world_size: int,
port: str,
hidden_q_full: int,
hidden_k_full: int,
seed: int,
eps: float,
) -> None:
"""Compare the ROCm AITER fusion with the eager TP implementation."""
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
world_size, 1, local_rank, port, local_rank=local_rank
)
try:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "AITER custom all-reduce was not initialized"
assert not aiter_ar.disabled, "AITER custom all-reduce is disabled"
assert hasattr(aiter_ar.aiter_ca, "custom_fused_qknorm_ar")
hq = hidden_q_full // world_size
hk = hidden_k_full // world_size
test_cases = (
(1, torch.bfloat16),
(128, torch.bfloat16),
(333, torch.bfloat16),
(1, torch.float16),
(128, torch.float16),
(333, torch.float16),
)
for num_tokens, dtype in test_cases:
q_norm = MiniMaxText01RMSNormTP(hidden_q_full, eps=eps).cuda()
k_norm = MiniMaxText01RMSNormTP(hidden_k_full, eps=eps).cuda()
set_random_seed(seed)
q_weight = torch.randn(hidden_q_full, dtype=dtype, device="cuda")
k_weight = torch.randn(hidden_k_full, dtype=dtype, device="cuda")
q_norm.weight = nn.Parameter(
q_weight[local_rank * hq : (local_rank + 1) * hq]
)
k_norm.weight = nn.Parameter(
k_weight[local_rank * hk : (local_rank + 1) * hk]
)
torch.manual_seed(seed + num_tokens + local_rank)
qkv = torch.randn(num_tokens, hq + hk + hk, dtype=dtype, device="cuda")
ref_q, ref_k = rms_norm_tp._minimax_qk_norm_tp_eager(
qkv.clone(),
q_norm.weight,
k_norm.weight,
hq,
hk,
world_size,
eps,
)
with (
patch.object(rms_norm_tp, "_MINIMAX_FUSED_AR_RMS_QK", None),
patch.object(
rms_norm_tp,
"_minimax_qk_norm_tp_fallback",
side_effect=AssertionError(
"ROCm AITER test unexpectedly used the fallback"
),
),
):
fused_q, fused_k, fused_v = MiniMaxText01RMSNormTP.forward_qkv(
q_norm, k_norm, qkv.clone(), hq, hk
)
torch.accelerator.synchronize()
torch.testing.assert_close(fused_q, ref_q, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(fused_k, ref_k, atol=3e-2, rtol=3e-2)
assert torch.equal(fused_v, qkv[:, hq + hk :])
finally:
cleanup_dist_env_and_memory()
@pytest.mark.distributed(num_gpus=2)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm required",
)
def test_minimax_reduce_rms_qk_rocm_aiter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The MiniMax TP QK path dispatches to AITER and matches eager RMSNorm."""
assert is_aiter_found_and_supported(), "AITER requires ROCm CDNA3 or newer"
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
world_size = 2
if current_platform.device_count() < world_size:
pytest.skip("Need at least two GPUs")
spawn(
_worker_forward_qk_rocm_aiter,
args=(world_size, str(get_open_port()), 6144, 1024, 42, 1e-6),
nprocs=world_size,
join=True,
)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA required",
)
@pytest.mark.parametrize("world_size", [2, 4, 8])
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
@pytest.mark.parametrize(
"hidden_dims",
[(6144, 1024)],
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.parametrize("seed", [42])
def test_minimax_reduce_rms_qk(
world_size,
num_tokens,
hidden_dims,
dtype,
eps,
seed,
):
num_gpus = current_platform.device_count()
if num_gpus < world_size:
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
hidden_q_full, hidden_k_full = hidden_dims
port = str(get_open_port())
spawn(
_worker_forward_qk,
args=(
world_size,
port,
num_tokens,
hidden_q_full,
hidden_k_full,
dtype,
seed,
eps,
),
nprocs=world_size,
join=True,
)
@pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="CUDA and Triton required",
)
@pytest.mark.parametrize("num_tokens", [1, 7, 128, 333, 2049])
@pytest.mark.parametrize("hidden_dims", [(3072, 512), (768, 256), (3000, 500)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("tp_world", [1, 4, 8])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.parametrize("seed", [42])
def test_minimax_qk_norm_triton_fallback(
monkeypatch, num_tokens, hidden_dims, dtype, tp_world, eps, seed
):
"""Single-GPU check: Triton fallback kernels vs the pure-torch reference.
The all-reduce is a TP communication barrier, so it is monkeypatched to
identity here; both the Triton path and the reference see the same
(patched) reduction. This validates the kernel math and the folded
``/ tp_world`` scaling without needing multiple ranks -- ``hidden_dims``
are the per-rank q/k segment widths.
"""
monkeypatch.setattr(rms_norm_tp, "_all_reduce_variance", lambda v: v)
q_size, kv_size = hidden_dims
device = "cuda"
torch.manual_seed(seed)
qkv = torch.randn(num_tokens, q_size + 2 * kv_size, dtype=dtype, device=device)
q_weight = torch.randn(q_size, dtype=dtype, device=device)
k_weight = torch.randn(kv_size, dtype=dtype, device=device)
q_triton, k_triton = rms_norm_tp._minimax_qk_norm_tp_fallback(
qkv, q_weight, k_weight, q_size, kv_size, 0, tp_world, eps
)
q_ref, k_ref = rms_norm_tp._minimax_qk_norm_tp_eager(
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
)
torch.testing.assert_close(q_triton, q_ref, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(k_triton, k_ref, atol=3e-2, rtol=3e-2)
-357
View File
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import pytest
import torch
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.mrope import (
apply_interleaved_rope,
triton_mrope,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.config import get_config
from vllm.utils.torch_utils import set_random_seed
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def generate_test_data(
num_tokens: int,
num_q_heads: int,
num_kv_heads: int,
head_size: int,
max_position_embeddings: int,
dtype: torch.dtype,
device: torch.device,
):
"""Generate test data for given configuration."""
set_random_seed(42)
# Create 2D positions (3, num_tokens) for multimodal case
positions = torch.randint(
0, max_position_embeddings // 4, (3, num_tokens), device=device
)
# Create query and key tensors
query = torch.randn(num_tokens, num_q_heads * head_size, dtype=dtype, device=device)
key = torch.randn(num_tokens, num_kv_heads * head_size, dtype=dtype, device=device)
return positions, query, key
class MRoPETestInfo(NamedTuple):
model_name: str
is_neox_style: bool = True
# https://github.com/pytorch/pytorch/blob/main/torch/testing/_comparison.py#L1317
atol: float = 1e-2
rtol: float = 1.6e-2
marks: list[pytest.MarkDecorator] = []
MODELS_TO_TEST = [
MRoPETestInfo(
model_name="zai-org/GLM-4.1V-9B-Thinking",
is_neox_style=False,
),
MRoPETestInfo(model_name="Qwen/Qwen2-VL-7B-Instruct"),
MRoPETestInfo(model_name="Qwen/Qwen2-VL-72B-Instruct"),
MRoPETestInfo(model_name="Qwen/Qwen2.5-VL-72B-Instruct"),
MRoPETestInfo(model_name="Qwen/Qwen3-VL-4B-Instruct"),
MRoPETestInfo(model_name="Qwen/Qwen3-VL-30B-A3B-Instruct"),
]
num_tokens_list = [11, 8192]
def test_apply_interleaved_rope():
mrope_section = [3, 1, 1]
x = torch.tensor(
[
[[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]],
[[10, 11, 12, 13, 14], [15, 16, 17, 18, 19]],
[[20, 21, 22, 23, 24], [25, 26, 27, 28, 29]],
]
)
result = apply_interleaved_rope(x, mrope_section)
expected = torch.tensor([[0, 11, 22, 3, 4], [5, 16, 27, 8, 9]])
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only test."
)
def test_apply_interleaved_rope_torch_compile():
mrope_section = [24, 20, 20]
num_tokens = 8192
rotary_dim = sum(mrope_section) * 2
cache = torch.randn(
3,
num_tokens,
rotary_dim,
device=device,
dtype=torch.bfloat16,
)
x = cache[..., : rotary_dim // 2]
expected = apply_interleaved_rope(x, mrope_section)
compiled_fn = torch.compile(
apply_interleaved_rope,
backend="inductor",
fullgraph=True,
)
result = compiled_fn(x, mrope_section)
torch.testing.assert_close(result, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
)
@pytest.mark.parametrize(
"model_info, model_name",
[
pytest.param(test_config, test_config.model_name, marks=test_config.marks)
for test_config in MODELS_TO_TEST
],
)
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_tokens", num_tokens_list)
def test_mrope(
default_vllm_config,
model_name: str,
model_info: MRoPETestInfo,
tp_size: int,
dtype: torch.dtype,
num_tokens: int,
):
atol = model_info.atol
rtol = model_info.rtol
config = get_config(model_name, False).get_text_config()
# get the model config
total_num_kv_heads = config.num_key_value_heads
total_num_heads = config.num_attention_heads
num_heads = total_num_heads // tp_size
num_kv_heads = max(1, total_num_kv_heads // tp_size)
head_dim = (
config.head_dim
if hasattr(config, "head_dim")
else config.hidden_size // total_num_heads
)
is_neox_style = model_info.is_neox_style
max_position = config.max_position_embeddings
mrope_helper_class = get_rope(
head_size=head_dim,
max_position=max_position,
is_neox_style=is_neox_style,
rope_parameters=config.rope_parameters,
dtype=dtype,
).to(device=device)
# create q k v input tensors
# create rotary pos emb input tensors
positions, query, key = generate_test_data(
num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device
)
query_native, key_native = mrope_helper_class.forward_native(
positions,
query.clone(),
key.clone(),
)
query_cuda, key_cuda = mrope_helper_class.forward_cuda(
positions,
query.clone(),
key.clone(),
)
torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol)
torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol)
# Strided q/k views of a packed buffer (the MiniMax M3 ViT usage):
# bitwise-identical to the contiguous run, neighboring columns untouched.
pad = 16
packed = torch.zeros(
num_tokens,
query.shape[1] + key.shape[1] + pad,
dtype=dtype,
device=device,
)
packed[:, : query.shape[1]] = query
packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key
sentinel = packed[:, -pad:].clone()
q_strided = packed[:, : query.shape[1]]
k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]]
mrope_helper_class.forward_cuda(positions, q_strided, k_strided)
torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0)
torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0)
torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
)
def test_mrope_row_offset_int64():
# MiniMax M3 ViT worst case: a max-size video item packs ~768k patches,
# and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241
# rows. Row-base math must be int64 or those rows wrap to negative
# offsets (illegal memory access at startup profiling).
set_random_seed(0)
n_qh, n_kh, hd, rd = 16, 16, 80, 78
mrope_section = [13, 13, 13]
row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row
num_tokens = 2**31 // row_stride + 8
free, _ = current_platform.mem_get_info()
if free < 2 * num_tokens * row_stride * 2:
pytest.skip("not enough free device memory")
packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device)
q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device)
k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device)
packed[:, : n_qh * hd] = q_fill
packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill
half_rd = rd // 2
cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
q = packed[:, : n_qh * hd]
k = packed[:, n_qh * hd : (n_qh + n_kh) * hd]
triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True)
t_end = mrope_section[0]
h_end = t_end + mrope_section[1]
for r in (0, num_tokens - 8, num_tokens - 1):
cos_row = torch.cat(
[cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]]
)
sin_row = torch.cat(
[sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]]
)
for fill, out in ((q_fill, q[r]), (k_fill, k[r])):
row = fill.view(-1, hd).float()
r1, r2 = row[:, :half_rd], row[:, half_rd:rd]
exp = torch.cat(
[r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]],
dim=1,
)
torch.testing.assert_close(
out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
)
@pytest.mark.parametrize(
"model_info, model_name",
[
pytest.param(test_config, test_config.model_name, marks=test_config.marks)
for test_config in MODELS_TO_TEST
],
)
@pytest.mark.parametrize("tp_size", [1, 2])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_tokens", num_tokens_list)
def test_mrope_torch_compile_tracing(
default_vllm_config,
model_name: str,
model_info: MRoPETestInfo,
tp_size: int,
dtype: torch.dtype,
num_tokens: int,
):
atol = model_info.atol
rtol = model_info.rtol
config = get_config(model_name, False).get_text_config()
# get the model config
total_num_kv_heads = config.num_key_value_heads
total_num_heads = config.num_attention_heads
num_heads = total_num_heads // tp_size
num_kv_heads = max(1, total_num_kv_heads // tp_size)
head_dim = (
config.head_dim
if hasattr(config, "head_dim")
else config.hidden_size // total_num_heads
)
is_neox_style = model_info.is_neox_style
max_position = config.max_position_embeddings
mrope_helper_class = get_rope(
head_size=head_dim,
max_position=max_position,
is_neox_style=is_neox_style,
rope_parameters=config.rope_parameters,
dtype=dtype,
).to(device=device)
# Generate test data
positions, query, key = generate_test_data(
num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device
)
# Create a wrapper that makes the in-place function appear functional
def functional_forward_cuda(pos, q, k):
"""Wrapper that converts in-place operation to functional style.
CUDA Graph does not support in-place operations.
This wrapper creates working copies of the
input tensors and modifies them.
"""
q_work = q.clone() # Create working copies
k_work = k.clone()
# Your in-place function modifies q_work and k_work
mrope_helper_class.forward_cuda(pos, q_work, k_work)
return q_work, k_work # Return the modified tensors
# Get reference results
query_native, key_native = mrope_helper_class.forward_native(
positions,
query.clone(),
key.clone(),
)
try:
compiled_forward_cuda = torch.compile(
functional_forward_cuda,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
# Run compiled version
query_compiled_cuda, key_compiled_cuda = compiled_forward_cuda(
positions,
query,
key,
)
# Run original version for comparison
query_cuda = query.clone()
key_cuda = key.clone()
mrope_helper_class.forward_cuda(positions, query_cuda, key_cuda)
# Verify results
torch.testing.assert_close(
query_compiled_cuda, query_cuda, atol=atol, rtol=rtol
)
torch.testing.assert_close(key_compiled_cuda, key_cuda, atol=atol, rtol=rtol)
torch.testing.assert_close(
query_compiled_cuda, query_native, atol=atol, rtol=rtol
)
torch.testing.assert_close(key_compiled_cuda, key_native, atol=atol, rtol=rtol)
print("✓ forward_cuda successfully traced with torch.compile inductor")
except Exception as e:
pytest.fail(f"forward_cuda failed to trace with torch.compile inductor: {e}")
-193
View File
@@ -1,193 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from itertools import product
import pytest
import torch
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.utils.torch_utils import set_random_seed
IS_NEOX_STYLE = [True, False]
DTYPES = [torch.bfloat16, torch.float]
HEAD_SIZES = [64, 80, 120, 256]
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
NUM_HEADS = [17] # Arbitrary values for testing
BATCH_SIZES = [5] # Arbitrary values for testing
SEQ_LENS = [11, 8192] # Arbitrary values for testing
SEEDS = [0]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
USE_KEY = [True, False]
def _get_flat_tensor_shape(
batch_size: int, seq_len: int, num_heads: int, head_size: int
) -> tuple[int, ...]:
return (batch_size, seq_len, num_heads * head_size)
# For testing sliced tensors
def _get_padded_tensor_shape(
batch_size: int, seq_len: int, num_heads: int, head_size: int
) -> tuple[int, ...]:
return (batch_size, seq_len, num_heads, head_size + 64)
def _get_batch_tensor_shape(
batch_size: int, seq_len: int, num_heads: int, head_size: int
) -> tuple[int, ...]:
return (batch_size, seq_len, num_heads, head_size)
TENSORS_SHAPES_FN = [
_get_batch_tensor_shape,
_get_flat_tensor_shape,
_get_padded_tensor_shape,
]
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@pytest.mark.parametrize("tensor_shape_fn", TENSORS_SHAPES_FN)
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
@pytest.mark.parametrize("seq_len", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("use_key", USE_KEY)
@torch.inference_mode()
def test_rotary_embedding(
default_vllm_config,
is_neox_style: bool,
tensor_shape_fn: Callable[[int, int, int, int], tuple[int, ...]],
batch_size: int,
seq_len: int,
num_heads: int,
head_size: int,
rotary_dim: int | None,
dtype: torch.dtype,
seed: int,
device: str,
use_key: bool,
max_position: int = 8192,
rope_theta: float = 10000,
) -> None:
if rotary_dim is None:
rotary_dim = head_size
set_random_seed(seed)
torch.set_default_device(device)
if rotary_dim is None:
rotary_dim = head_size
rope_parameters = {
"rope_type": "default",
"rope_theta": rope_theta,
"partial_rotary_factor": rotary_dim / head_size,
}
rope = get_rope(head_size, max_position, is_neox_style, rope_parameters)
rope = rope.to(dtype=dtype, device=torch.get_default_device())
positions = torch.randint(0, max_position, (batch_size, seq_len))
query_shape = tensor_shape_fn(batch_size, seq_len, num_heads, head_size)
# slice tensor if required, noop otherwise
query = torch.randn(query_shape, dtype=dtype)[..., :head_size]
key = torch.randn_like(query)[..., :head_size] if use_key else None
# NOTE(woosuk): The reference implementation should be executed first
# because the custom kernel is in-place.
ref_query, ref_key = rope.forward_native(positions, query, key)
out_query, out_key = rope.forward(positions, query, key)
# Compare the results.
torch.testing.assert_close(
out_query,
ref_query,
atol=get_default_atol(out_query),
rtol=get_default_rtol(out_query),
)
if use_key:
torch.testing.assert_close(
out_key,
ref_key,
atol=get_default_atol(out_key),
rtol=get_default_rtol(out_key),
)
else:
assert ref_key is None and out_key is None, "expected returned key to be None"
@torch.inference_mode()
def test_rope_module_cache(default_vllm_config):
MAX_POSITIONS = [123, 1234]
ROPE_THETAS = [10000, 1000000]
ROPE_PARAMETERS = (
{"rope_type": "default"},
{"rope_type": "linear", "factor": (1,)},
{"rope_type": "dynamic", "factor": 1},
)
settings = (
HEAD_SIZES,
ROTARY_DIMS,
MAX_POSITIONS,
ROPE_THETAS,
IS_NEOX_STYLE,
ROPE_PARAMETERS,
DTYPES,
)
rope_setting_id_map: dict[str, int] = {}
for setting in product(*settings):
(
head_size,
rotary_dim,
max_position,
rope_theta,
is_neox_style,
rope_parameters,
dtype,
) = setting
if rotary_dim is None:
rotary_dim = head_size
rope_parameters["rope_theta"] = rope_theta
rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
rope = get_rope(
head_size,
max_position,
is_neox_style,
rope_parameters,
dtype,
)
# different settings cannot share the same rope module
assert id(rope) not in rope_setting_id_map.values()
assert all(x.dtype == dtype for x in rope.buffers())
assert all(x.dtype == dtype for x in rope.parameters())
rope_setting_id_map[str(setting)] = id(rope)
for setting in product(*settings):
(
head_size,
rotary_dim,
max_position,
rope_theta,
is_neox_style,
rope_parameters,
dtype,
) = setting
if rotary_dim is None:
rotary_dim = head_size
rope_parameters["rope_theta"] = rope_theta
rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
rope = get_rope(
head_size,
max_position,
is_neox_style,
rope_parameters,
dtype,
)
# check if cache take effect
assert id(rope) == rope_setting_id_map[str(setting)]
-684
View File
@@ -1,684 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm AITER helper-op tests for CI stability.
These tests directly validate AITER kernel correctness against PyTorch/reference
implementations. They provide clear failure signals when AITER library updates
introduce regressions, independent of vLLM's fusion/compilation passes.
Tested ops:
- ``rms_norm`` and ``rms_norm2d_with_add`` vs PyTorch reference
- ``triton_rotary_embedding`` vs manual NeoX RoPE reference (xfail - known issue)
- ``act_mul_and_fp8_group_quant`` (SiGLU + FP8 group quant)
- fused RMSNorm + quantization ops vs sequential composition
Related coverage:
- per-token/per-tensor quant roundtrips: ``tests/rocm/aiter/test_quant_op_schema.py``
- RMSNorm determinism: ``tests/rocm/aiter/test_quant_op_schema.py``
- group_fp8_quant: ``tests/kernels/quantization/test_rocm_aiter_grouped_quant.py``
"""
import warnings
import pytest
import torch
import vllm._aiter_ops # noqa: F401 - ensure ops are registered
from tests.kernels.utils import bf16_ulp_distance, fp8_ulp_distance
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.fixture(autouse=True)
def setup_cuda_device():
"""Set default device to CUDA and seed RNG for all tests."""
torch.set_default_device("cuda")
set_random_seed(0)
def require_aiter():
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("aiter is not found or not supported on this hardware")
def require_fp8():
if not current_platform.supports_fp8():
pytest.skip("FP8 is not supported on this hardware")
def _format_observed_rate(count: int, total: int) -> str:
return f"{count / total:.4%} ({count}/{total})"
def _format_allowed_rate(rate: float, total: int) -> str:
allowed_count = int(rate * total)
return f"{rate:.4%} (<= {allowed_count}/{total})"
def _quantile(values: torch.Tensor, q: float) -> float:
if values.numel() == 0:
return 0.0
return torch.quantile(values, q).item()
def _dequantize_grouped(
quantized: torch.Tensor, scales: torch.Tensor, group_size: int
) -> torch.Tensor:
"""Expand per-group scales and dequantize to float32."""
M, N = quantized.shape
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :N]
return quantized.float() * scales_expanded
def _rms_norm_reference(
x: torch.Tensor, weight: torch.Tensor, eps: float
) -> torch.Tensor:
"""RMSNorm reference implementation in float32."""
rms = x.float().pow(2).mean(-1, keepdim=True).add(eps).sqrt()
return x.float() / rms * weight.float()
def _assert_quant_shapes(
x_quant: torch.Tensor,
scale: torch.Tensor,
M: int,
N: int,
fp8_dtype: torch.dtype,
num_groups: int | None = None,
) -> None:
"""Verify quantized output shapes and dtypes."""
assert x_quant.shape == (M, N)
assert x_quant.dtype == fp8_dtype
if num_groups is not None:
assert scale.shape == (M, num_groups)
else:
assert scale.shape == (M, 1)
def _assert_close_budget(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
atol: float,
rtol: float = 0.0,
pass_rate: float = 0.99999,
max_violation_factor: float = 3.0,
) -> None:
actual_f = actual.detach().float().flatten()
expected_f = expected.detach().float().flatten()
abs_diff = (actual_f - expected_f).abs()
allowed = atol + rtol * expected_f.abs()
total = abs_diff.numel()
passed = int((abs_diff <= allowed).sum().item())
failed = total - passed
allowed_fail_rate = 1.0 - pass_rate
max_abs = abs_diff.max().item()
mean_abs = abs_diff.mean().item()
p99_abs = _quantile(abs_diff, 0.99)
p999_abs = _quantile(abs_diff, 0.999)
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
max_atol = max_violation_factor * atol
above_max_count = int((abs_diff > max_atol).sum().item())
msg = (
"[rocm_aiter_ops] "
f"{label}: "
f"pass={passed / total:.4%} ({passed}/{total}) "
f"fail={_format_observed_rate(failed, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"atol={atol:g} "
f"rtol={rtol:g} "
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
f"max_abs={max_abs:.6g} "
f"mean_abs={mean_abs:.6g} "
f"p99_abs={p99_abs:.6g} "
f"p999_abs={p999_abs:.6g} "
f"worst_ratio={worst_ratio:.6g}"
)
print(msg)
if failed > 0:
warnings.warn(msg, stacklevel=2)
assert passed / total >= pass_rate, msg
assert max_abs <= max_atol, msg
assert mean_abs <= atol * 0.25, msg
def _assert_rel_error_quality(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
mean_limit: float,
preferred_rel: float,
max_rel: float,
max_fail_rate: float,
) -> None:
rel = (
(actual.float() - expected.float()).abs()
/ expected.float().abs().clamp_min(1e-5)
).flatten()
total = rel.numel()
within_preferred_count = int((rel <= preferred_rel).sum().item())
fail_count = total - within_preferred_count
above_max_count = int((rel > max_rel).sum().item())
mean_rel = rel.mean().item()
max_rel_err = rel.max().item()
p99 = _quantile(rel, 0.99)
p999 = _quantile(rel, 0.999)
msg = (
"[rocm_aiter_ops] "
f"{label}: "
f"rel<={preferred_rel:g} pass={within_preferred_count / total:.4%} "
f"({within_preferred_count}/{total}) "
f"fail={_format_observed_rate(fail_count, total)} "
f"mean_limit={mean_limit:.4%} "
f"rel>{max_rel:g}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
f"mean_rel={mean_rel:.6g} "
f"max_rel={max_rel_err:.6g} "
f"p99={p99:.6g} "
f"p999={p999:.6g}"
)
print(msg)
if fail_count > 0:
warnings.warn(msg, stacklevel=2)
assert mean_rel < mean_limit, msg
assert above_max_count / total <= max_fail_rate, msg
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_aiter_rms_norm_vs_torch(dtype):
"""rocm_aiter_rms_norm matches PyTorch manual RMSNorm for float16 and bfloat16."""
require_aiter()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 8, 128
x = torch.randn(M, N, dtype=dtype)
weight = torch.ones(N, dtype=dtype)
eps = 1e-5
# Reference: float32 RMSNorm for precision
ref = _rms_norm_reference(x, weight, eps).to(dtype)
out = rocm_aiter_ops.rms_norm(x, weight, eps)
# Allow max 1 ULP difference due to different accumulation order.
ulp = bf16_ulp_distance(out, ref)
max_ulp = int(ulp.max().item())
assert max_ulp <= 1, f"rms_norm dtype={dtype}: max ULP distance {max_ulp} > 1"
# -- Numerical accuracy tests for AITER custom ops -------------------------
def test_rocm_aiter_rmsnorm_with_add_vs_torch():
"""rocm_aiter_rmsnorm2d_fwd_with_add matches manual residual+RMSNorm reference."""
require_aiter()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 16, 256
x = torch.randn(M, N, dtype=torch.bfloat16)
residual = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.ones(N, dtype=torch.bfloat16)
eps = 1e-5
# Reference: add residual, then RMSNorm
h = x.float() + residual.float()
rms = h.pow(2).mean(-1, keepdim=True).add(eps).sqrt()
ref_normed = (h / rms * weight.float()).to(torch.bfloat16)
ref_residual = h.to(torch.bfloat16)
out, res_out = rocm_aiter_ops.rms_norm2d_with_add(x, residual, weight, eps)
# Allow max 1 ULP difference due to different accumulation order.
ulp_normed = bf16_ulp_distance(out, ref_normed)
max_ulp_normed = int(ulp_normed.max().item())
assert max_ulp_normed <= 1, (
f"rmsnorm_with_add normed: max ULP distance {max_ulp_normed} > 1"
)
ulp_residual = bf16_ulp_distance(res_out, ref_residual)
max_ulp_residual = int(ulp_residual.max().item())
assert max_ulp_residual <= 1, (
f"rmsnorm_with_add residual: max ULP distance {max_ulp_residual} > 1"
)
def test_rocm_aiter_triton_rotary_embedding_vs_torch():
"""rocm_aiter_triton_rotary_embedding matches manual NeoX-style RoPE reference.
The AITER kernel calls rope_cached_thd_positions_offsets_2c_fwd_inplace with
reuse_freqs_front_part=True. This means it reads only the first head_size//2
entries of cos and sin from the cache and applies them to both halves of the
head (pairwise NeoX rotation). The cos_sin_cache must be built with the second
half mirroring the first so the reference and kernel agree.
"""
require_aiter()
num_tokens = 8
num_heads = 4
head_size = 64
half_dim = head_size // 2
max_pos = 32
positions = torch.randint(0, max_pos, (num_tokens,), dtype=torch.long)
query = torch.randn(num_tokens, num_heads * head_size, dtype=torch.bfloat16)
key = torch.randn(num_tokens, num_heads * head_size, dtype=torch.bfloat16)
# Build cos/sin cache with second half mirroring first half.
# The kernel uses reuse_freqs_front_part=True: it reads only cos/sin[:, :half_dim]
# and applies those same frequencies to both the first and second
# halves of the head.
cos_half = torch.randn(max_pos, half_dim, dtype=torch.bfloat16)
sin_half = torch.randn(max_pos, half_dim, dtype=torch.bfloat16)
cos_cache = torch.cat([cos_half, cos_half], dim=-1) # [max_pos, head_size]
sin_cache = torch.cat([sin_half, sin_half], dim=-1) # [max_pos, head_size]
cos_sin_cache = torch.cat([cos_cache, sin_cache], dim=-1) # [max_pos, 2*head_size]
# Reference: NeoX-style pairwise rotation with front-half frequencies only.
# rotate_style=0 (NeoX): [x1*c - x2*s, x2*c + x1*s]
cos_pos = cos_half[positions] # [num_tokens, half_dim]
sin_pos = sin_half[positions] # [num_tokens, half_dim]
def apply_rope_ref(t: torch.Tensor) -> torch.Tensor:
# AITER triton keeps native dtype (no fp32 upcast), so reference must match.
t_r = t.view(num_tokens, num_heads, head_size)
c = cos_pos.unsqueeze(1) # [num_tokens, 1, half_dim]
s = sin_pos.unsqueeze(1)
x1, x2 = t_r[..., :half_dim], t_r[..., half_dim:]
rotated = torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], dim=-1)
return rotated.view(num_tokens, num_heads * head_size)
ref_q = apply_rope_ref(query)
ref_k = apply_rope_ref(key)
# AITER in-place RoPE (modifies query/key in-place)
q_aiter = query.clone()
k_aiter = key.clone()
torch.ops.vllm.rocm_aiter_triton_rotary_embedding(
positions,
q_aiter,
k_aiter,
head_size,
cos_sin_cache,
True, # is_neox style -> rotate_style=0
)
torch.testing.assert_close(q_aiter, ref_q)
torch.testing.assert_close(k_aiter, ref_k)
def test_rocm_aiter_act_mul_fp8_group_quant_roundtrip():
"""act_mul_and_fp8_group_quant: dequantized output matches SiLU gate reference."""
require_aiter()
require_fp8()
M, N = 32, 512 # N even: N//2 gate, N//2 up
group_size = 128
x = torch.randn(M, N, dtype=torch.bfloat16)
x_quant, scale = torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant(
x, group_size
)
N_half = N // 2
# Reference: SiLU(gate) * up
gate = x.float()[:, :N_half]
up = x.float()[:, N_half:]
ref = torch.sigmoid(gate) * gate * up # SiGLU
x_dequant = _dequantize_grouped(x_quant, scale, group_size)
_assert_rel_error_quality(
x_dequant,
ref,
label="act_mul_fp8_group_quant",
mean_limit=0.1,
preferred_rel=0.1,
max_rel=1.0,
max_fail_rate=0.01,
)
# -- Fused RMSNorm + quantization accuracy tests ---------------------------
def test_rocm_aiter_rmsnorm_fused_dynamic_quant_vs_sequential():
"""Fused RMSNorm+per-token-FP8-quant matches sequential rms_norm->per_token_quant.
Tests that the fused kernel produces the same result as the two-step
sequential composition: rms_norm(x) followed by per_token_quant.
The fused path is used in production for inference throughput.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 32, 512
x = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.ones(N, dtype=torch.bfloat16)
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
# Sequential reference
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
ref_q, ref_scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
ref_dequant = ref_q.float() * ref_scale.float()
# Fused op
fused_q, fused_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant(
x, weight, eps, fp8_dtype
)
_assert_quant_shapes(fused_q, fused_scale, M, N, fp8_dtype)
fused_dequant = fused_q.float() * fused_scale.float()
# Fused vs sequential: both should recover RMSNorm output within FP8 error
_assert_rel_error_quality(
fused_dequant,
ref_dequant,
label="rmsnorm_fused_dynamic_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
def test_rocm_aiter_rmsnorm_fused_add_dynamic_quant_vs_reference():
"""Fused (residual-add + RMSNorm + per-token-FP8-quant)
matches sequential reference.
Production path: input + residual -> RMSNorm -> FP8 quantize, returning
both the quantized norm output and the residual sum for the next layer.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 16, 256
x = torch.randn(M, N, dtype=torch.bfloat16)
residual = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.ones(N, dtype=torch.bfloat16)
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
# Sequential reference: add residual -> rms_norm -> per_token_quant
h = (x.float() + residual.float()).to(torch.bfloat16)
normed = rocm_aiter_ops.rms_norm(h, weight, eps)
ref_q, ref_scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
ref_residual_out = h
# Fused op: returns (x_quant, residual_out, scale)
fused_q, fused_res_out, fused_scale = (
torch.ops.vllm.rocm_aiter_rmsnorm_fused_add_dynamic_quant(
x, residual, weight, eps, fp8_dtype
)
)
_assert_quant_shapes(fused_q, fused_scale, M, N, fp8_dtype)
assert fused_res_out.shape == (M, N)
torch.testing.assert_close(fused_res_out, ref_residual_out, atol=1e-2, rtol=1.6e-2)
# Dequantized output matches sequential path
fused_dequant = fused_q.float() * fused_scale.float()
ref_dequant = ref_q.float() * ref_scale.float()
_assert_rel_error_quality(
fused_dequant,
ref_dequant,
label="rmsnorm_fused_add_dynamic_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
def test_rocm_aiter_rmsnorm_fp8_group_quant_vs_sequential():
"""Fused RMSNorm+FP8-group-quant matches sequential rms_norm->group_fp8_quant.
Tests both output shapes and dequantized accuracy against the two-step
sequential composition.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 32, 512
group_size = 128
x = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.ones(N, dtype=torch.bfloat16)
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
expected_groups = (N + group_size - 1) // group_size
# Fused op: (x_quant, scales)
fused_q, fused_scales = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
x, weight, eps, group_size
)
_assert_quant_shapes(fused_q, fused_scales, M, N, fp8_dtype, expected_groups)
# Dequantize and compare to reference: rms_norm -> group quant -> dequant
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed, group_size)
ref_dequant = _dequantize_grouped(ref_q, ref_scales, group_size)
fused_dequant = _dequantize_grouped(fused_q, fused_scales, group_size)
_assert_rel_error_quality(
fused_dequant,
ref_dequant,
label="rmsnorm_fp8_group_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
def test_rocm_aiter_rmsnorm_with_add_fp8_group_quant_residual_accuracy():
"""Fused rmsnorm_with_add_fp8_group_quant residual output matches x + residual."""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 16, 256
group_size = 128
x = torch.randn(M, N, dtype=torch.bfloat16)
residual = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.ones(N, dtype=torch.bfloat16)
eps = 1e-5
fused_q, fused_res, fused_scales = (
torch.ops.vllm.rocm_aiter_rmsnorm_with_add_fp8_group_quant(
x, residual, weight, eps, group_size
)
)
# Residual output must equal x + residual
ref_residual = (x.float() + residual.float()).to(torch.bfloat16)
_assert_close_budget(
fused_res.float(),
ref_residual.float(),
label="rmsnorm_with_add_fp8_group_quant residual",
atol=1e-2,
rtol=1e-2,
)
# Dequantized quant output must match rms_norm(x + residual)
h = ref_residual
ref_normed = _rms_norm_reference(h, weight, eps).to(torch.bfloat16)
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(ref_normed, group_size)
ref_dequant = _dequantize_grouped(ref_q, ref_scales, group_size)
fused_dequant = _dequantize_grouped(fused_q, fused_scales, group_size)
_assert_rel_error_quality(
fused_dequant,
ref_dequant,
label="rmsnorm_with_add_fp8_group_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
@pytest.mark.parametrize("m", [1, 32])
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
"""Fused q/kv RMSNorm + q group-quant matches rms_norm -> group_fp8_quant.
M=1 covers the single-token decode case. M=0 is guarded by the caller
(the ROCm DeepSeek-V4 attention checks num_tokens > 0) and the op has
no zero-row guard itself, so it is intentionally not tested here.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
nq, nk, group_size = 1536, 576, 128
q = torch.randn(m, nq, dtype=torch.bfloat16)
kv = torch.randn(m, nk, dtype=torch.bfloat16)
q_weight = torch.ones(nq, dtype=torch.bfloat16)
kv_weight = torch.ones(nk, dtype=torch.bfloat16)
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
fused_q, fused_scales, fused_kv = rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
)
_assert_quant_shapes(fused_q, fused_scales, m, nq, fp8_dtype, nq // group_size)
assert fused_kv.shape == (m, nk)
assert fused_kv.dtype == kv.dtype
# q: dequantized comparison against the sequential composition.
normed_q = rocm_aiter_ops.rms_norm(q, q_weight, eps)
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed_q, group_size)
_assert_rel_error_quality(
_dequantize_grouped(fused_q, fused_scales, group_size),
_dequantize_grouped(ref_q, ref_scales, group_size),
label="mla_dual_rmsnorm_group_quant",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
# kv: both sides are aiter RMSNorm kernels; allow one bf16 ULP.
ref_kv = rocm_aiter_ops.rms_norm(kv, kv_weight, eps)
max_ulp = int(bf16_ulp_distance(fused_kv, ref_kv).max().item())
assert max_ulp <= 1, f"fused kv: max ULP {max_ulp} > 1"
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
contiguous inputs."""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
m, nq, nk, group_size = 16, 1536, 576, 128
qr_kv = torch.randn(m, nq + nk, dtype=torch.bfloat16)
q_view, kv_view = qr_kv.split([nq, nk], dim=-1)
assert not q_view.is_contiguous()
q_weight, kv_weight = torch.ones(nq + nk, dtype=torch.bfloat16).split([nq, nk])
eps = 1e-5
def run(q, kv):
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
)
out_strided = run(q_view, kv_view)
out_contig = run(q_view.contiguous(), kv_view.contiguous())
q_ulp = int(fp8_ulp_distance(out_strided[0], out_contig[0]).max().item())
kv_ulp = int(bf16_ulp_distance(out_strided[2], out_contig[2]).max().item())
assert q_ulp <= 1, f"strided q: max fp8 ULP {q_ulp} > 1"
assert kv_ulp <= 1, f"strided kv: max bf16 ULP {kv_ulp} > 1"
torch.testing.assert_close(out_strided[1], out_contig[1])
# -- End-to-end inference chain test ---------------------------------------
def test_rocm_aiter_rms_norm_then_per_token_quant_e2e():
"""End-to-end: BF16 RMSNorm -> per-token FP8 quantization -> dequantize.
Simulates the inference path through a transformer layer norm before a
linear projection: verifies the full chain produces accurate output
compared to a float32 reference.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
# Llama-style hidden dim
M, N = 32, 4096
x = torch.randn(M, N, dtype=torch.bfloat16)
weight = torch.randn(N, dtype=torch.bfloat16) # learned scale
eps = 1e-5
fp8_dtype = current_platform.fp8_dtype()
# Float32 reference for the full chain
ref_normed_f32 = _rms_norm_reference(x, weight, eps)
# AITER chain: RMSNorm -> per-token FP8 quant -> dequant
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
x_q, scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
x_dequant = x_q.float() * scale.float() # scale: [M, 1]
_assert_quant_shapes(x_q, scale, M, N, fp8_dtype)
# Dequantized result should match the float32 reference within FP8 quant error
_assert_rel_error_quality(
x_dequant,
ref_normed_f32,
label="rms_norm_then_per_token_quant_e2e",
mean_limit=0.05,
preferred_rel=0.05,
max_rel=0.5,
max_fail_rate=0.01,
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_aiter_rms_norm_then_group_fp8_quant_e2e(dtype):
"""End-to-end: RMSNorm (fp16/bf16) -> FP8 group quantization -> dequantize.
Covers both float16 and bfloat16 inputs to verify dtype-agnostic
behavior of the RMSNorm+group-quant pipeline.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
M, N = 16, 512
group_size = 128
x = torch.randn(M, N, dtype=dtype)
weight = torch.randn(N, dtype=dtype)
eps = 1e-5
# Float32 reference
ref_normed_f32 = _rms_norm_reference(x, weight, eps)
# AITER chain: RMSNorm -> group FP8 quant -> dequant
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
x_fp8, scales = rocm_aiter_ops.group_fp8_quant(normed.bfloat16(), group_size)
x_dequant = _dequantize_grouped(x_fp8, scales, group_size)
_assert_rel_error_quality(
x_dequant,
ref_normed_f32,
label=f"rms_norm_then_group_fp8_quant_e2e dtype={dtype}",
mean_limit=0.1,
preferred_rel=0.1,
max_rel=1.0,
max_fail_rate=0.01,
)
-226
View File
@@ -1,226 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm platform and extension smoke tests.
These tests catch ROCm-specific regressions early:
1. **Extension import smoke tests**: Verify that ROCm native extensions load
without crashing. Build/linking issues surface here before runtime.
2. **GCN arch parsing tests**: Verify _capability_from_gcn_arch correctly
parses all known GPU architectures. Wrong parsing causes silent feature
gate failures or crashes from selecting unsupported kernels.
3. **AITER ops availability tests**: Verify rocm_aiter_ops accessor methods
work without crashing, regardless of hardware support.
4. **Env var propagation tests**: Verify ROCm-specific env vars correctly
gate their respective features (e.g., shuffle KV cache, skinny GEMM).
Related coverage:
- AITER kernel numerics: ``tests/kernels/core/test_rocm_aiter_ops.py``
- Attention backend selection: ``tests/v1/attention/test_rocm_attention_backends_selection.py``
- GEMM dispatch: ``tests/model_executor/layers/test_rocm_unquantized_gemm.py``
""" # noqa: E501
import importlib
import pytest
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
# ---------------------------------------------------------------------------
# Extension import smoke tests
# ---------------------------------------------------------------------------
def test_rocm_extension_imports():
"""ROCm native extension loads without error.
Catches build/linking issues before runtime. If this fails, the ROCm
build is broken and nothing else will work.
"""
import vllm._rocm_C # noqa: F401
def test_aiter_ops_module_imports():
"""AITER ops module imports without error.
The _aiter_ops module registers custom ops and initializes AITER state.
Import failures here indicate missing dependencies or incompatible
AITER versions.
"""
import vllm._aiter_ops # noqa: F401
def test_rocm_platform_module_imports():
"""ROCm platform module imports without error.
The platform module queries hardware via amdsmi and sets up arch flags.
Import failures indicate missing amdsmi or HIP runtime issues.
"""
import vllm.platforms.rocm # noqa: F401
# ---------------------------------------------------------------------------
# GCN arch parsing tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize(
("gcn_arch", "expected_major", "expected_minor"),
[
pytest.param("gfx90a", 9, 0, id="MI200-gfx90a"),
pytest.param("gfx942", 9, 4, id="MI300X-gfx942"),
pytest.param("gfx950", 9, 5, id="MI350-gfx950"),
pytest.param("gfx1100", 11, 0, id="RX7900-gfx1100"),
pytest.param("gfx1101", 11, 0, id="RX7800-gfx1101"),
pytest.param("gfx1151", 11, 5, id="StrixHalo-gfx1151"),
pytest.param("gfx1200", 12, 0, id="RDNA4-gfx1200"),
pytest.param("gfx1201", 12, 0, id="RX9070-gfx1201"),
pytest.param("gfx1250", 12, 5, id="CDNA5-gfx1250"),
],
)
def test_capability_from_gcn_arch_known_gpus(gcn_arch, expected_major, expected_minor):
"""_capability_from_gcn_arch parses known GPU architectures correctly.
This is critical for feature gating - wrong parsing causes:
- FP8 kernels selected on hardware that doesn't support them
- MX format support incorrectly enabled/disabled
- Custom allreduce used on unsupported hardware
"""
from vllm.platforms.rocm import _capability_from_gcn_arch
result = _capability_from_gcn_arch(gcn_arch)
assert result is not None, f"Failed to parse {gcn_arch}"
assert result == (expected_major, expected_minor), (
f"Wrong capability for {gcn_arch}: got {result}, "
f"expected ({expected_major}, {expected_minor})"
)
@pytest.mark.parametrize(
"gcn_arch",
[
pytest.param("not_a_gfx_string", id="non-gfx-string"),
pytest.param("cuda_arch", id="cuda-style"),
pytest.param("", id="empty-string"),
],
)
def test_capability_from_gcn_arch_returns_none_for_non_gfx(gcn_arch):
"""_capability_from_gcn_arch returns None for non-GFX strings.
Non-GFX strings should return None so the caller can fall back to
torch.cuda.get_device_capability.
"""
from vllm.platforms.rocm import _capability_from_gcn_arch
result = _capability_from_gcn_arch(gcn_arch)
assert result is None
@pytest.mark.parametrize(
"gcn_arch",
[
pytest.param("gfx8", id="too-few-digits"),
pytest.param("gfx12345", id="too-many-digits"),
pytest.param("gfx700", id="unsupported-major-7"),
pytest.param("gfx1500", id="future-major-15"),
],
)
def test_capability_from_gcn_arch_raises_for_malformed(gcn_arch):
"""_capability_from_gcn_arch raises ValueError for malformed GFX strings.
Malformed strings that look like GFX but can't be parsed should raise
so the user gets a clear error message asking them to file an issue.
"""
from vllm.platforms.rocm import _capability_from_gcn_arch
with pytest.raises(ValueError, match="GCN arch"):
_capability_from_gcn_arch(gcn_arch)
# ---------------------------------------------------------------------------
# AITER ops availability tests
# ---------------------------------------------------------------------------
def test_rocm_aiter_ops_accessor_methods_dont_crash():
"""rocm_aiter_ops accessor methods work without crashing.
These methods gate kernel selection. Even if AITER isn't available,
they should return False gracefully rather than crashing.
"""
from vllm._aiter_ops import rocm_aiter_ops
# These should all return bool without crashing
assert isinstance(rocm_aiter_ops.is_enabled(), (bool, type(None)))
assert isinstance(rocm_aiter_ops.is_linear_enabled(), (bool, type(None)))
assert isinstance(rocm_aiter_ops.is_fused_moe_enabled(), (bool, type(None)))
assert isinstance(rocm_aiter_ops.is_mla_enabled(), (bool, type(None)))
assert isinstance(rocm_aiter_ops.is_mha_enabled(), (bool, type(None)))
def test_rocm_aiter_ops_is_aiter_found():
"""is_aiter_found returns consistent results."""
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found
# The cached global should match the function result
assert is_aiter_found() == IS_AITER_FOUND
def test_rocm_aiter_ops_refresh_env_variables_doesnt_crash():
"""refresh_env_variables works without crashing.
This is called after monkeypatching env vars in tests. It should
never crash, even with unusual env var combinations.
"""
from vllm._aiter_ops import rocm_aiter_ops
# Should not raise
rocm_aiter_ops.refresh_env_variables()
# ---------------------------------------------------------------------------
# Env var propagation tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("enabled", [True, False])
def test_shuffle_kv_cache_env_propagates_to_rocm_aiter_ops(enabled, monkeypatch):
"""VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT propagates to AITER state."""
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", "1" if enabled else "0")
importlib.reload(envs)
rocm_aiter_ops.refresh_env_variables()
assert envs.VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT is enabled
assert rocm_aiter_ops.is_shuffle_kv_cache_enabled() is enabled
@pytest.mark.parametrize("enabled", [True, False])
def test_skinny_gemm_env_controls_rocm_fp8_scaled_mm_support(enabled, monkeypatch):
"""VLLM_ROCM_USE_SKINNY_GEMM gates ROCm FP8 scaled-mm kernel eligibility."""
import vllm.envs as envs
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
ROCmFP8ScaledMMLinearKernel,
)
monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "1" if enabled else "0")
importlib.reload(envs)
monkeypatch.setattr("vllm.platforms.rocm.on_mi3xx", lambda: True)
supported, reason = ROCmFP8ScaledMMLinearKernel.is_supported()
assert supported is enabled
if enabled:
assert reason is None
else:
assert reason == "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled."
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for miscellaneous utilities."""
import pytest
import torch
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
def rotary_embedding_opcheck(
rot,
positions: torch.Tensor,
query: torch.Tensor,
key: torch.Tensor | None = None,
):
cos_sin_cache = rot.cos_sin_cache.to(query.device, dtype=query.dtype)
# ops.rotary_embedding() is a in-place operation
# that updates the query and key tensors.
opcheck(
torch.ops._C.rotary_embedding,
(positions, query, key, rot.head_size, cos_sin_cache, rot.is_neox_style),
)
@pytest.mark.parametrize("device", ["cuda"])
@pytest.mark.parametrize("max_position", [11, 4096, 32768])
@pytest.mark.parametrize("is_neox_style", [True, False])
@pytest.mark.parametrize("rotary_dim", [32])
@pytest.mark.parametrize("head_size", [32, 108])
@pytest.mark.parametrize("seq_len", [11, 1024])
@pytest.mark.parametrize("use_key", [True, False])
@pytest.mark.parametrize("head_stride_is_contiguous", [True, False])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_rotary_embedding_opcheck(
default_vllm_config,
dist_init,
device,
max_position,
is_neox_style,
rotary_dim,
head_size,
seq_len,
use_key,
head_stride_is_contiguous,
dtype,
):
batch_size = 1
base = 10000
num_heads = 7
rot = RotaryEmbedding(
head_size, rotary_dim, max_position, base, is_neox_style, dtype
)
positions = torch.randint(0, max_position, (batch_size, seq_len), device=device)
head_stride = head_size + (64 if head_stride_is_contiguous else 0)
query = torch.randn(
batch_size, seq_len, num_heads, head_stride, dtype=dtype, device=device
)
key = torch.randn_like(query) if use_key else None
query = query[..., :head_size]
key = key[..., :head_size] if key is not None else None
rotary_embedding_opcheck(rot, positions, query, key)
# if we have a contiguous head stride, test the alternate
# [..., num_heads * head_dim] shape/layout
if head_stride_is_contiguous:
rotary_embedding_opcheck(
rot,
positions,
query.flatten(start_dim=-2),
key.flatten(start_dim=-2) if key is not None else None,
)
@@ -1,325 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for fused MLA KV-cache write and RoPE fused kernel."""
import random
import pytest
import torch
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
from vllm import _custom_ops as ops
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
def assert_rope_close_rocm(
actual: torch.Tensor,
ref: torch.Tensor,
rope_input: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
is_neox: bool,
*,
max_fail_fraction: float | None = None,
atol: float | None = None,
rtol: float | None = None,
) -> None:
"""assert_close for RoPE fused-vs-reference on ROCm, with a per-element
product-ULP allowance for catastrophic-cancellation elements.
The fused and reference kernels may contract the RoPE multiply-add into an
FMA with either product pre-rounded to the working dtype. The two valid
orderings differ by at most 2 ULPs of the product at that element, which
cancellation (x*cos ~= -y*sin) can amplify past the default tolerance.
Tier 1: |actual - ref| <= atol + rtol * |ref| (default tolerance).
Tier 2: elements failing tier 1 must each satisfy
|actual_e - ref_e| <= 2 * eps * |product_e|, where product_e is the larger
of the two products combined at that element, and their fraction must be
<= max_fail_fraction (1e-4 for fp16; 1e-2 for bf16, whose coarser eps
puts 1-ULP divergences past the default tolerance far more often).
"""
if atol is None:
atol = get_default_atol(ref)
if rtol is None:
rtol = get_default_rtol(ref)
if max_fail_fraction is None:
max_fail_fraction = 1e-2 if ref.dtype == torch.bfloat16 else 1e-4
eps = torch.finfo(ref.dtype).eps
# per-element product magnitude, in the (seq, heads, rot_dim) layout
seq_len, rot_dim = rope_input.shape[0], rope_input.shape[-1]
half = rot_dim // 2
inp = rope_input.reshape(seq_len, -1, rot_dim).to(torch.float32)
cs = cos_sin_cache[positions].to(torch.float32) # (seq_len, 2*half)
cos = cs[:, :half].unsqueeze(1) # broadcast over heads
sin = cs[:, half:].unsqueeze(1)
x, y = inp[..., :half], inp[..., half:]
if is_neox:
# out = [x*cos - y*sin, y*cos + x*sin]
prod = torch.cat(
[
torch.maximum((x * cos).abs(), (y * sin).abs()),
torch.maximum((y * cos).abs(), (x * sin).abs()),
],
dim=-1,
)
else:
# out[2i] = x[2i]*cos[i] - x[2i+1]*sin[i], interleaved
xe, xo = inp[..., 0::2], inp[..., 1::2]
prod = torch.stack(
[
torch.maximum((xe * cos).abs(), (xo * sin).abs()),
torch.maximum((xo * cos).abs(), (xe * sin).abs()),
],
dim=-1,
).flatten(-2)
prod = prod.reshape(rope_input.shape)
diff = (actual.to(torch.float32) - ref.to(torch.float32)).abs()
tol = atol + rtol * ref.to(torch.float32).abs()
fail = diff > tol
n_fail = int(fail.sum())
if n_fail == 0:
return
assert n_fail <= max_fail_fraction * diff.numel(), (
f"{n_fail} elements beyond default tolerance "
f"({n_fail / diff.numel():.2%} > {max_fail_fraction:.2%}); "
f"max diff {diff.max().item()}"
)
excess = diff[fail] - 2 * eps * prod[fail]
assert (excess <= 0).all(), (
f"{int((excess > 0).sum())} cancellation elements exceed 2 product-ULPs; "
f"worst excess {excess.max().item()}"
)
@pytest.fixture
def default_vllm_config(monkeypatch):
"""Enable the AITER triton rope on ROCm for fp16-consistent numerics.
The fused CUDA kernel runs native fp16 while forward_native upcasts to
fp32, so on ROCm we route through the AITER triton rope (+rotary_embedding)
to match. Its env gates are cached at import, hence refresh_env_variables().
"""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
is_rocm = current_platform.is_rocm()
if is_rocm:
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["+rotary_embedding"])
)
else:
config = VllmConfig()
try:
with monkeypatch.context() as m, set_current_vllm_config(config):
if is_rocm:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_TRITON_ROPE", "1")
rocm_aiter_ops.refresh_env_variables()
yield config
finally:
if is_rocm:
rocm_aiter_ops.refresh_env_variables()
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16, torch.float])
@pytest.mark.parametrize("is_neox_style", [False, True])
@pytest.mark.parametrize("seq_len", [11, 42])
@pytest.mark.parametrize("qk_rope_head_dim", [64, 128])
@pytest.mark.parametrize("num_q_heads", [128])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("num_blocks", [64])
@pytest.mark.parametrize("block_size", [16, 64, 256])
@pytest.mark.parametrize("seed", [0])
@pytest.mark.parametrize(
"device",
[f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)],
)
@torch.inference_mode()
def test_concat_and_cache_mla_rope_fused(
default_vllm_config,
dtype: torch.dtype,
is_neox_style: bool,
seq_len: int,
qk_rope_head_dim: int,
num_q_heads: int,
kv_cache_dtype: str,
kv_lora_rank: int,
num_blocks: int,
block_size: int,
seed: int,
device: str,
max_position: int = 8192,
base: float = 10000,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
rope = RotaryEmbedding(
qk_rope_head_dim,
qk_rope_head_dim,
max_position,
base,
is_neox_style,
torch.float32,
)
rope = rope.to(dtype=dtype, device=torch.get_default_device())
positions = torch.randint(0, max_position, (seq_len,))
query = torch.randn(seq_len, num_q_heads, qk_rope_head_dim, dtype=dtype)
key = torch.randn(seq_len, 1, qk_rope_head_dim + kv_lora_rank, dtype=dtype)
k_pe = torch.flatten(key[..., :qk_rope_head_dim], start_dim=1).to(device=device)
kv_c = torch.flatten(key[..., qk_rope_head_dim:], start_dim=1).to(device=device)
k_pe_orig = k_pe.clone()
query_orig = query.clone()
if current_platform.is_rocm():
# We use forward_hip for the same numerics as the fused custom kernel on ROCm
# when dtype is FP16. The torch-native implementation implicitly upcasts
# FP16 x FP16 multiplications to FP32 before downcasting them, which leads
# to notable output divergences.
# Clone the tensors because the implementation modifies them in-place
ref_q_pe, ref_k_pe = rope.forward_hip(positions, query.clone(), k_pe.clone())
else:
# NOTE(woosuk): The reference implementation should be executed first
# because the custom kernel is in-place.
ref_q_pe, ref_k_pe = rope.forward_native(positions, query, k_pe)
assert ref_k_pe is not None
ref_k_pe = torch.flatten(ref_k_pe, start_dim=1).to(device=device)
ref_k_rope = ref_k_pe[..., :qk_rope_head_dim]
total_available_slots = num_blocks * block_size
total_needed_slots = seq_len
assert total_available_slots >= total_needed_slots, "Not enough kv slots!"
slot_mapping_lst = random.sample(range(total_available_slots), total_needed_slots)
slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device)
entry_size = kv_lora_rank + qk_rope_head_dim
kv_cache_scale = torch.tensor([0.1], dtype=torch.float32, device=device)
kv_cache = torch.zeros(
num_blocks,
block_size,
entry_size,
dtype=torch.uint8 if kv_cache_dtype == "fp8" else dtype,
device=device,
)
ref_temp = torch.zeros(*kv_cache.shape, dtype=dtype, device=device)
for i in range(seq_len):
slot = slot_mapping[i].item()
block_idx = slot // block_size
block_offset = slot % block_size
ref_temp[block_idx, block_offset] = torch.cat((kv_c[i], ref_k_rope[i]), -1)
if kv_cache_dtype == "fp8":
ref_kv_cache = torch.empty_like(ref_temp, dtype=kv_cache.dtype)
ops.convert_fp8(
ref_kv_cache, ref_temp, kv_cache_scale.item(), kv_dtype=kv_cache_dtype
)
else:
ref_kv_cache = ref_temp
opcheck(
torch.ops._C_cache_ops.concat_and_cache_mla_rope_fused,
(
positions,
query,
k_pe,
kv_c,
rope.cos_sin_cache,
is_neox_style,
slot_mapping,
kv_cache,
kv_cache_dtype,
kv_cache_scale,
),
test_utils=DEFAULT_OPCHECK_TEST_UTILS,
)
ops.concat_and_cache_mla_rope_fused(
positions,
query,
k_pe,
kv_c,
rope.cos_sin_cache,
is_neox_style,
slot_mapping,
kv_cache,
kv_cache_dtype,
kv_cache_scale,
)
# On ROCm the AITER Triton rope diverges by up to 2 product-ULPs from the
# fused kernel, which the default tolerance misses at the rare
# catastrophic-cancellation elements. fp8 keeps its calibrated tolerance.
rocm_neox = current_platform.is_rocm() and is_neox_style
rocm_bf16 = current_platform.is_rocm() and dtype == torch.bfloat16
if kv_cache_dtype == "fp8":
result_temp = torch.empty_like(kv_cache, dtype=torch.float16)
ops.convert_fp8(
result_temp,
kv_cache.contiguous(),
kv_cache_scale.item(),
kv_dtype=kv_cache_dtype,
)
expected_temp = torch.empty_like(ref_kv_cache, dtype=torch.float16)
ops.convert_fp8(
expected_temp, ref_kv_cache, kv_cache_scale.item(), kv_dtype=kv_cache_dtype
)
torch.testing.assert_close(
result_temp,
expected_temp,
atol=0.004 if rocm_bf16 else 0.001,
rtol=0.15 if rocm_neox or rocm_bf16 else 0.1,
)
elif current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
# gather the rope region of the cache back to token order
block_idx = slot_mapping // block_size
block_offset = slot_mapping % block_size
fused_k_rope = kv_cache[block_idx, block_offset, kv_lora_rank:]
ref_k_rope_gathered = ref_kv_cache[block_idx, block_offset, kv_lora_rank:]
assert_rope_close_rocm(
fused_k_rope,
ref_k_rope_gathered,
k_pe_orig,
rope.cos_sin_cache,
positions,
is_neox_style,
)
# the nope region is a byte-exact copy
torch.testing.assert_close(
kv_cache[..., :kv_lora_rank], ref_kv_cache[..., :kv_lora_rank]
)
else:
torch.testing.assert_close(kv_cache, ref_kv_cache)
if current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
assert_rope_close_rocm(
query,
ref_q_pe,
query_orig,
rope.cos_sin_cache,
positions,
is_neox_style,
)
else:
torch.testing.assert_close(
query,
ref_q_pe,
atol=get_default_atol(query),
rtol=get_default_rtol(query),
)
-292
View File
@@ -1,292 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
from vllm.v1.worker.gpu import buffer_utils
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor
DEVICE_TYPE = current_platform.device_type
DEVICES = [
f"{DEVICE_TYPE}:{i}"
for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", DEVICES)
def test_cpu_write(device):
torch.set_default_device(device)
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
# as the UVA view's ops never trigger lazy init.
torch.accelerator.set_device_index(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 0
assert gpu_view[4, 5] == 0
cpu_tensor[0, 0] = 1
cpu_tensor[2, 3] = 2
cpu_tensor[4, 5] = -1
gpu_view.mul_(2)
torch.accelerator.synchronize(gpu_view.device)
assert gpu_view[0, 0] == 2
assert gpu_view[2, 3] == 4
assert gpu_view[4, 5] == -2
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", DEVICES)
def test_gpu_write(device):
torch.set_default_device(device)
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
# as the UVA view's ops never trigger lazy init.
torch.accelerator.set_device_index(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 0
assert gpu_view[4, 5] == 0
gpu_view[0, 0] = 1
gpu_view[2, 3] = 2
gpu_view[4, 5] = -1
gpu_view.mul_(2)
torch.accelerator.synchronize(gpu_view.device)
assert cpu_tensor[0, 0] == 2
assert cpu_tensor[2, 3] == 4
assert cpu_tensor[4, 5] == -2
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", DEVICES)
def test_staged_write_uses_uva_contents_for_uva_target(device, monkeypatch):
def fail_async_tensor_h2d(*args, **kwargs):
pytest.fail("UVA-backed targets should not copy write contents to the GPU")
monkeypatch.setattr(buffer_utils, "async_tensor_h2d", fail_async_tensor_h2d)
staged = StagedWriteTensor(
(3, 4096),
dtype=torch.int32,
device=torch.device(device),
max_concurrency=2,
uva_instead_of_gpu=True,
)
staged.stage_write(2, 3, [11, 12, 13])
staged.apply_write()
torch.accelerator.synchronize()
staged.stage_write(1, 7, [21, 22])
staged.apply_write()
torch.accelerator.synchronize()
staged.stage_write(0, 1020, range(1500))
staged.apply_write()
torch.accelerator.synchronize()
assert staged.gpu[2, 3:6].tolist() == [11, 12, 13]
assert staged.gpu[1, 7:9].tolist() == [21, 22]
assert staged.gpu[0, 1020:2520].tolist() == list(range(1500))
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("input_type", ["list", "numpy", "tensor"])
@pytest.mark.parametrize("size", [(4,), (4, 3)])
def test_uva_pool_overwrites_exposed_prefix(input_type, size):
"""Both slots expose only current contents across growth and shorter reuse."""
pool = buffer_utils.UvaBufferPool(size, torch.int32, max_concurrency=2)
for buf in pool._uva_bufs:
assert tuple(buf.cpu.shape) == size
assert buf.cpu.is_pinned()
assert torch.count_nonzero(buf.cpu).item() == 0
lengths = [0, 0, 3, 3, 4, 4, 5, 3, 3, 5, 1024, 1024, 1025, 1025, 2, 2]
for step, length in enumerate(lengths):
if input_type == "list" and len(size) > 1 and length == 0:
# An empty list has no trailing shape; preserve NumPy's rejection.
with pytest.raises(ValueError, match="could not broadcast"):
pool.copy_to_uva([])
continue
shape = (length, *size[1:])
expected = (
torch.arange(int(np.prod(shape)), dtype=torch.int32, device="cpu").reshape(
shape
)
- step
)
values = expected.tolist()
if input_type == "numpy":
values = expected.numpy()
elif input_type == "tensor":
values = expected
before = list(pool._uva_bufs)
slot = (pool._curr + 1) % pool.max_concurrency
result = pool.copy_to_uva(values)
assert tuple(result.shape) == shape
assert pool._curr == slot
assert pool._uva_bufs[1 - slot] is before[1 - slot]
if length <= before[slot].cpu.shape[0]:
assert pool._uva_bufs[slot] is before[slot]
else:
assert tuple(pool._uva_bufs[slot].cpu.shape) == (
1 << (length - 1).bit_length(),
*size[1:],
)
assert pool.size == size
# Blocking read also retires GPU readers before the next slot reuse.
torch.testing.assert_close(result.cpu(), expected, rtol=0, atol=0)
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("use_out", [False, True])
@pytest.mark.parametrize("input_type", ["numpy", "tensor"])
def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type):
pool = buffer_utils.UvaBufferPool((2, 3), torch.int32, max_concurrency=2)
for length in (2, 5, 3, 6):
expected = torch.arange(length * 3, dtype=torch.int32, device="cpu").reshape(
length, 3
)
values = expected.numpy() if input_type == "numpy" else expected
out = (
torch.empty(expected.shape, dtype=torch.int32, device=DEVICE_TYPE)
if use_out
else None
)
result = pool.copy_to_gpu(values, out=out)
if use_out:
assert result is out
torch.testing.assert_close(result.cpu(), expected, rtol=0, atol=0)
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("uva_target", [False, True])
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64, torch.float32])
def test_staged_write_inflight(uva_target, dtype):
"""Preserve every generation until its consumer finishes before slot reuse."""
device = torch.device(f"{DEVICE_TYPE}:0")
with torch.accelerator.device_index(device.index):
state = StagedWriteTensor(
(4, 4096),
dtype,
device,
max_concurrency=2,
uva_instead_of_gpu=uva_target,
)
assert (state.write_contents is not None) == uva_target
stream = torch.Stream(device=device)
stream.wait_stream(torch.accelerator.current_stream())
pending: list[tuple[torch.Event, torch.Tensor, torch.Tensor]] = []
expected = torch.zeros((4, 4096), dtype=dtype, device="cpu")
for step in range(24):
if len(pending) == 2:
event, snapshot, reference = pending.pop(0)
event.synchronize()
torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0)
# Growing and shrinking lengths exercise reallocation and reuse.
length = [3, 17, 1500, 4090][step % 4]
row = step % 3
values = torch.arange(length, dtype=dtype, device="cpu") + step * 8192
if dtype == torch.float32:
values += 0.25
expected[row, 2 : 2 + length] = values
expected[3, 1:4] = step
with stream:
state.stage_write(row, 2, values.tolist())
state.stage_write(3, 1, [step] * 3)
state.apply_write()
# A GPU consumer observes this generation before the next update.
snapshot = state.gpu.clone()
event = torch.Event()
event.record(stream)
pending.append((event, snapshot, expected.clone()))
for event, snapshot, reference in pending:
event.synchronize()
torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0)
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.parametrize("device", DEVICES)
def test_non_pinned_cpu_tensor(device):
# Non-pinned CPU tensors are internally copied into a pinned buffer,
# so the resulting gpu view reflects the values at creation time but
# is decoupled from further writes to the original `cpu_tensor`.
torch.set_default_device(device)
cpu_tensor = torch.arange(100, dtype=torch.int32, device="cpu").view(10, 10)
assert not cpu_tensor.is_pinned()
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view[0, 0] == 0
assert gpu_view[2, 3] == 23
assert gpu_view[9, 9] == 99
# Writes to the original (unpinned) CPU tensor must not affect the view,
# since a private pinned copy was made.
cpu_tensor[0, 0] = -1
assert gpu_view[0, 0] == 0
# The view itself remains writable and independently usable.
gpu_view.mul_(2)
assert gpu_view[2, 3] == 46
assert gpu_view[9, 9] == 198
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only sync check.")
def test_non_uva_buffer_copies_without_stream_sync():
buffer = buffer_utils.NonUvaBuffer(8, torch.int32)
buffer.np[:] = np.arange(8)
torch.cuda.set_sync_debug_mode("error")
try:
prefix = buffer.uva(4)
full = buffer.uva()
finally:
torch.cuda.set_sync_debug_mode(0)
# The next step overwrites the host buffer right after the copy is issued.
buffer.np[:] = -1
torch.accelerator.synchronize()
assert prefix.tolist() == list(range(4))
assert full.tolist() == list(range(8))
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
@pytest.mark.skipif(
not current_platform.is_xpu(), reason="XPU non-contiguous UVA test."
)
@pytest.mark.parametrize("pinned", [False, True])
@pytest.mark.parametrize("device", DEVICES)
def test_non_contiguous_strided_view(device, pinned):
torch.set_default_device(device)
# Simulate scale_kn: a [32, 16] contiguous tensor
scale_kn = torch.arange(
512, dtype=torch.float32, device="cpu", pin_memory=pinned
).view(32, 16)
# Transposed view: shape [16, 32], non-contiguous stride (1, 16)
cpu_view = scale_kn.t()
assert cpu_view.shape == (16, 32)
assert cpu_view.stride() == (1, 16)
assert not cpu_view.is_contiguous()
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_view)
assert gpu_view.device.type == DEVICE_TYPE
assert gpu_view.shape == (16, 32)
assert gpu_view.stride() == (1, 16)
assert not gpu_view.is_contiguous()
# Transposing tensor_view back should yield a contiguous [32, 16] tensor
gpu_view_t = gpu_view.t()
assert gpu_view_t.shape == (32, 16)
assert gpu_view_t.is_contiguous()
# Correctness check: values match original transposed CPU tensor
assert torch.equal(gpu_view.cpu(), cpu_view)
-126
View File
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the stride-aware FP8 quantization kernel with head_dim padding."""
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
if HAS_TRITON:
from vllm.kernels.triton.qkv_padded_fp8_quant import (
quantize_fp8_pad_head_dim_triton,
)
HEAD_DIMS = [72, 80, 128]
SEQ_LENS = [64, 256]
NUM_HEADS = [16]
SCALES = [0.01, 0.1, 1.0]
def _naive_fp8_quantize(
tensor: torch.Tensor, scale: torch.Tensor, skip_scale: bool
) -> torch.Tensor:
"""Reference FP8 quantization in PyTorch."""
fp8_dtype = current_platform.fp8_dtype()
fp8_min, fp8_max = get_fp8_min_max()
x = tensor.float()
if not skip_scale:
x = x / scale.item()
x = x.clamp(fp8_min, fp8_max)
return x.to(fp8_dtype)
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("head_dim", HEAD_DIMS)
@pytest.mark.parametrize("seq_len", SEQ_LENS)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("scale_val", SCALES)
def test_quantize_contiguous(
head_dim: int, seq_len: int, num_heads: int, scale_val: float
) -> None:
"""Test quantization of contiguous 3D tensors."""
torch.manual_seed(42)
tensor = torch.randn(
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
scale = torch.tensor([scale_val], dtype=torch.float32, device="cuda").view(
1, 1, 1, 1
)
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
padded_dim = (head_dim + 15) // 16 * 16
assert result.shape == (seq_len, num_heads, padded_dim)
assert result.is_contiguous()
assert result.dtype == current_platform.fp8_dtype()
# Compare unpadded portion against reference
ref = _naive_fp8_quantize(tensor, scale, skip_scale=False)
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
# Padded region should be zero
if padded_dim > head_dim:
assert (result[:, :, head_dim:].float() == 0).all()
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
@pytest.mark.parametrize("head_dim", [72, 80])
def test_quantize_non_contiguous(head_dim: int) -> None:
"""Test quantization from non-contiguous QKV views (interleaved buffer)."""
seq_len, num_heads = 64, 16
# Simulate interleaved QKV buffer: shape (seq_len, 3 * num_heads, head_dim)
qkv = torch.randn(
seq_len, 3 * num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
# Q is every 3rd head slice - non-contiguous view
q = qkv[:, 0::3, :]
assert not q.is_contiguous()
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result = quantize_fp8_pad_head_dim_triton(q, scale)
padded_dim = (head_dim + 15) // 16 * 16
assert result.shape == (seq_len, num_heads, padded_dim)
assert result.is_contiguous()
# Compare against contiguous reference
ref = _naive_fp8_quantize(q.contiguous(), scale, skip_scale=False)
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
def test_skip_scale() -> None:
"""Test skip_scale=True produces cast-only output (no division)."""
seq_len, num_heads, head_dim = 32, 8, 80
tensor = torch.randn(
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
)
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result_skip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=True)
result_noskip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=False)
# skip_scale should just cast, not divide
ref_cast = _naive_fp8_quantize(tensor, scale, skip_scale=True)
torch.testing.assert_close(result_skip[:, :, :head_dim].float(), ref_cast.float())
# With scale != 1.0, skip and no-skip should differ
assert not torch.equal(result_skip.float(), result_noskip.float())
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
def test_4d_input() -> None:
"""Test that 4D input (B, S, H, D) is handled correctly."""
B, S, H, D = 2, 32, 8, 72
tensor = torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16)
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
padded_dim = (D + 15) // 16 * 16
assert result.shape == (B, S, H, padded_dim)
-296
View File
@@ -1,296 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FP8 scaling (dynamic and static) in MMEncoderAttention."""
import contextlib
import json
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.model_executor.layers.attention.mm_encoder_attention import (
_FP8_AMAX_HISTORY_LEN,
_FP8_MAX,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
is_flashinfer_cudnn_fp8_prefill_attn_supported,
)
pytestmark = pytest.mark.skipif(
not is_flashinfer_cudnn_fp8_prefill_attn_supported(),
reason="FlashInfer cuDNN FP8 prefill attention not supported",
)
LAYER_0 = "visual.blocks.0.attn.attn"
LAYER_1 = "visual.blocks.1.attn.attn"
NUM_HEADS = 16
HEAD_DIM = 72
def _is_mi3xx() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_mi3xx
return on_mi3xx()
@contextlib.contextmanager
def _build_attention(mm_config, attn_backend=None):
"""Yield an MMEncoderAttention with the given multimodal config.
The VllmConfig context stays active while the test runs so that
``get_multimodal_config()`` calls during the forward path resolve. Also
invokes ``process_weights_after_loading`` to simulate the model loader's
auto-scan. Yields ``None`` if FlashInfer cuDNN is not available.
"""
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if attn_backend is None:
attn_backend = AttentionBackendEnum.FLASHINFER
if (
attn_backend == AttentionBackendEnum.FLASHINFER
and not is_flashinfer_cudnn_fp8_prefill_attn_supported()
):
yield None
return
vllm_config = VllmConfig()
vllm_config.model_config = MagicMock(spec=ModelConfig, multimodal_config=mm_config)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=attn_backend,
),
):
attn = MMEncoderAttention(
num_heads=NUM_HEADS,
head_size=HEAD_DIM,
prefix=LAYER_0,
)
attn.process_weights_after_loading(torch.bfloat16)
yield attn
@pytest.fixture
def _make_attention():
"""Create an MMEncoderAttention with dynamic FP8 scaling."""
from vllm.config.multimodal import MultiModalConfig
with _build_attention(MultiModalConfig(mm_encoder_attn_dtype="fp8")) as attn:
yield attn
@pytest.fixture
def _make_static_attention(tmp_path):
"""Create an MMEncoderAttention with static FP8 scales from a file."""
from vllm.config.multimodal import MultiModalConfig
scale_file = tmp_path / "scales.json"
scale_file.write_text(
json.dumps(
{
LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0},
LAYER_1: {"q": 100.0, "k": 110.0, "v": 120.0},
}
)
)
with _build_attention(
MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
) as attn:
yield attn
def test_dynamic_scaling_updates_scales(_make_attention) -> None:
"""Verify that _record_amax_and_update_scales updates scale buffers."""
attn = _make_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
attn = attn.to("cuda")
S, H, D = 32, NUM_HEADS, HEAD_DIM
q = torch.full((S, H, D), 2.0, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), 3.0, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), 4.0, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
expected_q_scale = 2.0 / _FP8_MAX
expected_k_scale = 3.0 / _FP8_MAX
expected_v_scale = 4.0 / _FP8_MAX
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_q_scale)
torch.testing.assert_close(attn._fp8_k_scale.item(), expected_k_scale)
torch.testing.assert_close(attn._fp8_v_scale.item(), expected_v_scale)
def test_circular_buffer_wraps(_make_attention) -> None:
"""Verify the amax circular buffer wraps at HISTORY_LEN."""
attn = _make_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
attn = attn.to("cuda")
S, H, D = 16, NUM_HEADS, HEAD_DIM
for i in range(_FP8_AMAX_HISTORY_LEN + 2):
mag = float(i + 1)
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
assert attn._fp8_amax_pos == 2
expected_max = float(_FP8_AMAX_HISTORY_LEN + 2)
expected_scale = expected_max / _FP8_MAX
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_scale)
def test_static_scales_loaded(_make_static_attention) -> None:
"""Verify static scales are loaded from the JSON file."""
attn = _make_static_attention
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available (FlashInfer backend required)")
assert attn.fp8_enabled
assert not attn._fp8_dynamic_scale
# Layer 0 scales (the layer this attention was created with).
assert attn._fp8_q_scale.item() == 224.0
assert attn._fp8_k_scale.item() == 198.0
assert attn._fp8_v_scale.item() == 210.0
assert not attn.skip_scale_q
assert not attn.skip_scale_k
assert not attn.skip_scale_v
# No amax history buffers for static scaling.
assert not hasattr(attn, "_fp8_q_amax")
@pytest.mark.skipif(
not _is_mi3xx(),
reason="AITER FP8 attention requires MI300/MI350",
)
def test_aiter_static_scales_loaded(tmp_path) -> None:
"""Verify AITER reuses the existing static FP8 scale loading path."""
from vllm.config.multimodal import MultiModalConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
scale_file = tmp_path / "aiter_scales.json"
scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}}))
mm_config = MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn:
assert attn is not None
assert attn.fp8_enabled
assert not attn._fp8_dynamic_scale
assert attn._fp8_q_scale.item() == 224.0
assert attn._fp8_k_scale.item() == 198.0
assert attn._fp8_v_scale.item() == 210.0
def test_static_scales_missing_layer(tmp_path) -> None:
"""Verify error when requested layer is not in the scale file."""
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config.multimodal import MultiModalConfig
from vllm.v1.attention.backends.registry import AttentionBackendEnum
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
pytest.skip("FlashInfer cuDNN not available")
scale_file = tmp_path / "wrong_layer.json"
scale_file.write_text(
json.dumps({"visual.blocks.99.attn": {"q": 1.0, "k": 1.0, "v": 1.0}})
)
mm_config = MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_path=str(scale_file),
)
vllm_config = VllmConfig()
vllm_config.model_config = MagicMock(spec=ModelConfig, multimodal_config=mm_config)
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
with (
set_current_vllm_config(vllm_config),
patch(
"vllm.model_executor.layers.attention.mm_encoder_attention"
".get_vit_attn_backend",
return_value=AttentionBackendEnum.FLASHINFER,
),
):
attn = MMEncoderAttention(
num_heads=NUM_HEADS,
head_size=HEAD_DIM,
prefix=LAYER_0,
)
with pytest.raises(ValueError, match="scales not found for layer"):
attn.process_weights_after_loading(torch.bfloat16)
def test_dynamic_scales_auto_save(tmp_path) -> None:
"""Verify scales are saved to disk after the amax buffer fills."""
import vllm.model_executor.layers.attention.mm_encoder_attention as _mod
from vllm.config.multimodal import MultiModalConfig
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
pytest.skip("FlashInfer cuDNN not available")
# Reset module-level state between runs (other tests may have left
# state behind after triggering a save).
_mod._fp8_scale_save_path = None
_mod._fp8_saved_scale_refs.clear()
save_file = tmp_path / "auto_scales.json"
with _build_attention(
MultiModalConfig(
mm_encoder_attn_dtype="fp8",
mm_encoder_fp8_scale_save_path=str(save_file),
)
) as attn:
if attn is None or not attn.fp8_enabled:
pytest.skip("FP8 attention not available")
attn = attn.to("cuda")
S, H, D = 16, NUM_HEADS, HEAD_DIM
# Run exactly _FP8_AMAX_HISTORY_LEN forward passes.
for i in range(_FP8_AMAX_HISTORY_LEN):
mag = float(i + 1)
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
k = torch.full((S, H, D), mag * 0.5, device="cuda", dtype=torch.bfloat16)
v = torch.full((S, H, D), mag * 0.3, device="cuda", dtype=torch.bfloat16)
attn._record_amax_and_update_scales(q, k, v)
# File should have been written on the 16th call (buffer wrap).
assert save_file.is_file(), "Scale file was not saved"
scales = json.loads(save_file.read_text())
assert LAYER_0 in scales
assert set(scales[LAYER_0].keys()) == {"q", "k", "v"}
for val in scales[LAYER_0].values():
assert isinstance(val, float) and val > 0
# Path is cleared after the one-shot save fires.
assert _mod._fp8_scale_save_path is None
@@ -1,168 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the fused vocab-parallel embedding kernel
(``vllm._custom_ops.vocab_parallel_embedding``).
The guarantee: the kernel is a drop-in for the TP > 1 embedding path, i.e. it
returns bit-exactly what mask + shift + gather + ``masked_fill_`` returns on
each rank, and summing the per-rank outputs (what the all-reduce does)
reconstructs a full-table lookup.
"""
from unittest.mock import Mock
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
get_masked_input_and_mask,
pad_vocab_size,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
requires_cuda = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="the fused vocab-parallel embedding kernel is CUDA-only",
)
def _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size, pad=64):
return VocabParallelEmbedding._get_indices(
pad_vocab_size(vocab_size, pad),
pad_vocab_size(org_vocab_size, pad),
vocab_size,
org_vocab_size,
tp_rank,
tp_size,
)
def _reference(input_ids, weight, si):
"""The path the kernel replaces."""
masked_input, input_mask = get_masked_input_and_mask(
input_ids,
si.org_vocab_start_index,
si.org_vocab_end_index,
si.num_org_vocab_padding,
si.added_vocab_start_index,
si.added_vocab_end_index,
)
out = torch.nn.functional.embedding(masked_input.long(), weight)
return out.masked_fill_(input_mask.unsqueeze(-1), 0)
def _fused(input_ids, weight, si):
return ops.vocab_parallel_embedding(
input_ids,
weight,
si.org_vocab_start_index,
si.org_vocab_end_index,
si.num_org_vocab_padding,
si.added_vocab_start_index,
si.added_vocab_end_index,
)
def _shard(full_table, si, rows_per_rank, dtype):
"""Lay a rank's rows out the way the weight loader does."""
weight = torch.zeros(rows_per_rank, full_table.shape[1], dtype=dtype, device="cuda")
num_org = si.org_vocab_end_index - si.org_vocab_start_index
weight[:num_org] = full_table[si.org_vocab_start_index : si.org_vocab_end_index]
added_at = num_org + si.num_org_vocab_padding
num_added = si.added_vocab_end_index - si.added_vocab_start_index
weight[added_at : added_at + num_added] = full_table[
si.added_vocab_start_index : si.added_vocab_end_index
]
return weight
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("tp_size", [2, 4, 8])
@pytest.mark.parametrize(
"vocab_size,org_vocab_size", [(128256, 128256), (32256, 32000)]
)
@pytest.mark.parametrize("hidden", [4096, 3584])
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
"""Every rank's partial output is bit-exact with the eager path, including
the added-vocab (LoRA) rows and the ids owned by another rank."""
set_random_seed(7)
dtype = torch.bfloat16
rows = pad_vocab_size(vocab_size, 64) // tp_size
full_table = torch.randn(pad_vocab_size(vocab_size, 64), hidden, dtype=dtype).cuda()
ids = torch.randint(0, vocab_size, (257,), dtype=id_dtype, device="cuda")
for tp_rank in range(tp_size):
si = _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size)
weight = _shard(full_table, si, rows, dtype)
torch.testing.assert_close(
_fused(ids, weight, si), _reference(ids, weight, si), atol=0.0, rtol=0.0
)
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("tp_size", [2, 8])
def test_rank_sum_is_full_lookup(tp_size):
"""What the all-reduce sees: the partials add up to a full-table lookup."""
set_random_seed(11)
vocab_size, org_vocab_size, hidden = 32256, 32000, 2048
dtype = torch.bfloat16
rows = pad_vocab_size(vocab_size, 64) // tp_size
full_table = torch.randn(pad_vocab_size(vocab_size, 64), hidden, dtype=dtype).cuda()
# Ids from both the base and the added (LoRA) range.
ids = torch.randint(0, vocab_size, (129,), dtype=torch.int32, device="cuda")
ids[:16] = torch.randint(
org_vocab_size, vocab_size, (16,), dtype=torch.int32, device="cuda"
)
total = torch.zeros(ids.shape[0], hidden, dtype=dtype, device="cuda")
for tp_rank in range(tp_size):
si = _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size)
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("use_fused", [False, True])
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
"""SP's deferred reduction preserves masked contributions and their dtype."""
with torch.device("cuda"):
values = torch.arange(128 * 16).reshape(128, 16)
table = ((values % 29 - 14) / 16).to(dtype)
ids = torch.tensor([0, 63, 64, 127, 12])
expected = table[ids].float()
for rank in range(2):
local = expected.clone()
local[ids // 64 != rank] = 0
remote = (expected - local).to(dtype)
group = Mock(world_size=2, rank_in_group=rank)
# Model the other vocabulary owner's contribution, including FP8 bytes.
group.all_reduce.side_effect = lambda output, remote=remote: (
output + remote.view(output.dtype)
)
for reduce_results in (True, False):
embedding = VocabParallelEmbedding(
128,
16,
params_dtype=dtype,
parallel_group=group,
reduce_results=reduce_results,
)
embedding.use_fused_embedding = use_fused
embedding.weight_loader(embedding.get_parameter("weight"), table)
actual = embedding(ids)
assert actual.dtype == dtype
torch.testing.assert_close(
actual.float(),
expected if reduce_results else local,
atol=0,
rtol=0,
)