[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,567 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import random
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.model_executor.layers.activation import (
|
||||
FastGELU,
|
||||
FatreluAndMul,
|
||||
GeluAndMul,
|
||||
MulAndSilu,
|
||||
NewGELU,
|
||||
QuickGELU,
|
||||
ReLUSquaredActivation,
|
||||
SiluAndMul,
|
||||
SiluAndMulWithClamp,
|
||||
SwigluOAIAndMul,
|
||||
SwigluStepAndMul,
|
||||
swiglustep_and_mul_triton,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16, torch.float]
|
||||
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
|
||||
D = [512, 13824] # Arbitrary values for testing
|
||||
SEEDS = [0]
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
|
||||
def test_masked_moe_activation_rejects_unsupported_activation() -> None:
|
||||
input = torch.empty(1, 1, 2)
|
||||
output = torch.empty(1, 1, 1)
|
||||
valid_token_counts = torch.ones(1, dtype=torch.int32)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="relu2"):
|
||||
apply_moe_activation(
|
||||
MoEActivation.RELU2,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
|
||||
def test_moe_silu_clamp_uses_native_xpu_fallback(
|
||||
default_vllm_config, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(current_platform, "is_xpu", lambda: True)
|
||||
clamp_limit = 3.0
|
||||
input = torch.tensor([[12.0, -12.0, 8.0, -8.0], [-2.0, 2.0, -4.0, 4.0]])
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
apply_moe_activation(
|
||||
MoEActivation.SILU,
|
||||
output,
|
||||
input,
|
||||
activation_config=ApplyMoEActivationConfig(clamp_limit=clamp_limit),
|
||||
)
|
||||
|
||||
expected = SiluAndMulWithClamp(clamp_limit, compile_native=False).forward_native(
|
||||
input
|
||||
)
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def _assert_masked_moe_activation(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
mask_layout: str,
|
||||
d: int,
|
||||
max_num_tokens: int,
|
||||
) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts = 4
|
||||
input_dim = 2 * d if activation.is_gated else d
|
||||
if mask_layout == "flat":
|
||||
input = torch.randn(max_num_tokens, input_dim, dtype=dtype, device=device)
|
||||
valid_token_counts = torch.tensor(
|
||||
[max_num_tokens // 2], dtype=torch.int32, device=device
|
||||
)
|
||||
output = torch.full((max_num_tokens, d), 42.0, dtype=dtype, device=device)
|
||||
else:
|
||||
input = torch.randn(
|
||||
num_experts, max_num_tokens, input_dim, dtype=dtype, device=device
|
||||
)
|
||||
valid_token_counts = torch.tensor(
|
||||
[0, 1, max_num_tokens // 2, max_num_tokens],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_experts, max_num_tokens, d), 42.0, dtype=dtype, device=device
|
||||
)
|
||||
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
activation_config=activation_config,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
batched_input = input.view(-1, max_num_tokens, input_dim)
|
||||
batched_output = output.view(-1, max_num_tokens, d)
|
||||
for expert, num_tokens in enumerate(valid_token_counts.cpu().tolist()):
|
||||
if num_tokens:
|
||||
expected = torch.empty((num_tokens, d), dtype=dtype, device=device)
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
expected,
|
||||
batched_input[expert, :num_tokens].clone(),
|
||||
activation_config=activation_config,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
batched_output[expert, :num_tokens],
|
||||
expected,
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.all(batched_output[expert, num_tokens:] == 42.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
"silu_and_mul",
|
||||
"mul_and_silu",
|
||||
"gelu",
|
||||
"gelu_tanh",
|
||||
"fatrelu",
|
||||
"swigluoai_and_mul",
|
||||
"swiglustep_and_mul",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("d", D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_act_and_mul(
|
||||
default_vllm_config,
|
||||
activation: str,
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
|
||||
if activation == "silu_and_mul":
|
||||
layer = SiluAndMul(compile_native=False)
|
||||
fn = torch.ops._C.silu_and_mul
|
||||
if activation == "mul_and_silu":
|
||||
layer = MulAndSilu()
|
||||
fn = torch.ops._C.mul_and_silu
|
||||
elif activation == "gelu":
|
||||
layer = GeluAndMul(approximate="none")
|
||||
fn = torch.ops._C.gelu_and_mul
|
||||
elif activation == "gelu_tanh":
|
||||
layer = GeluAndMul(approximate="tanh")
|
||||
fn = torch.ops._C.gelu_tanh_and_mul
|
||||
elif activation == "fatrelu":
|
||||
threshold = random.uniform(0, 1)
|
||||
layer = FatreluAndMul(threshold)
|
||||
fn = torch.ops._C.fatrelu_and_mul
|
||||
elif activation == "swigluoai_and_mul":
|
||||
layer = SwigluOAIAndMul()
|
||||
fn = torch.ops._C.swigluoai_and_mul
|
||||
elif activation == "swiglustep_and_mul":
|
||||
layer = SwigluStepAndMul()
|
||||
fn = swiglustep_and_mul_triton
|
||||
out = layer(x)
|
||||
ref_out = layer.forward_native(x)
|
||||
if activation in ["swigluoai_and_mul", "swiglustep_and_mul"]:
|
||||
rtol = {
|
||||
# For fp16, change the relative tolerance from 1e-3 to 2e-3
|
||||
torch.float16: 2e-3,
|
||||
torch.bfloat16: 2e-2,
|
||||
torch.float: 1.3e-6,
|
||||
}
|
||||
|
||||
def _get_rtol(output) -> float:
|
||||
return rtol[output.dtype]
|
||||
|
||||
torch.testing.assert_close(
|
||||
out, ref_out, atol=get_default_atol(out), rtol=_get_rtol(out)
|
||||
)
|
||||
else:
|
||||
# The SiluAndMul, MulAndSilu, GELU and FatReLU implementations are
|
||||
# equivalent to the native PyTorch implementations, so we can do exact
|
||||
# comparison.
|
||||
torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
|
||||
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
if activation == "fatrelu":
|
||||
opcheck(fn, (out, x, threshold))
|
||||
elif activation == "swigluoai_and_mul":
|
||||
opcheck(fn, (out, x, layer.alpha, layer.limit))
|
||||
elif activation != "swiglustep_and_mul":
|
||||
opcheck(fn, (out, x))
|
||||
|
||||
|
||||
SWIGLU_LIMITS = [3.0, 7.0, 15.0]
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_swiglu_limit_func_without_routing_uses_output_buffer() -> None:
|
||||
x = torch.randn(7, 1024, dtype=torch.bfloat16, device="cuda")
|
||||
output = torch.empty(7, 512, dtype=x.dtype, device=x.device)
|
||||
|
||||
swiglu_limit_func(output, x, swiglu_limit=7.0)
|
||||
gate, up = x.chunk(2, dim=-1)
|
||||
expected = torch.nn.functional.silu(gate.clamp(max=7.0)) * up.clamp(
|
||||
min=-7.0, max=7.0
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("alpha", "beta"),
|
||||
[(1.0, 0.0), (1.702, 0.0), (1.0, 1.0)],
|
||||
)
|
||||
@pytest.mark.parametrize("swiglu_limit", SWIGLU_LIMITS)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("d", D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_silu_and_mul_with_clamp(
|
||||
default_vllm_config,
|
||||
alpha: float,
|
||||
beta: float,
|
||||
swiglu_limit: float,
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
"""SiluAndMulWithClamp: cuda kernel must match native reference."""
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
# Use large values to ensure clamping is exercised.
|
||||
x = torch.randn(num_tokens, 2 * d, dtype=dtype) * swiglu_limit * 2
|
||||
|
||||
default_vllm_config.compilation_config.custom_ops = [
|
||||
"none",
|
||||
"+silu_and_mul_with_clamp",
|
||||
]
|
||||
layer = SiluAndMulWithClamp(
|
||||
swiglu_limit,
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
compile_native=False,
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
# forward_hip is always dispatched; the alpha/beta gate is checked
|
||||
# inside it at call time rather than picked at construction time, so
|
||||
# verify the actual routing by spying on the two candidate methods.
|
||||
assert layer._forward_method == layer.forward_hip
|
||||
with (
|
||||
patch.object(layer, "forward_cuda", wraps=layer.forward_cuda) as cuda_spy,
|
||||
patch.object(
|
||||
layer, "forward_native", wraps=layer.forward_native
|
||||
) as native_spy,
|
||||
):
|
||||
out = layer(x)
|
||||
if alpha == 1.0 and beta == 0.0:
|
||||
cuda_spy.assert_called_once()
|
||||
native_spy.assert_not_called()
|
||||
else:
|
||||
native_spy.assert_called_once()
|
||||
cuda_spy.assert_not_called()
|
||||
else:
|
||||
assert layer._forward_method == layer.forward_cuda
|
||||
out = layer(x)
|
||||
|
||||
ref_out = layer.forward_native(x)
|
||||
|
||||
rtol = {
|
||||
torch.float16: 2e-3,
|
||||
torch.bfloat16: 2e-2,
|
||||
torch.float: 1.3e-6,
|
||||
}
|
||||
torch.testing.assert_close(
|
||||
out, ref_out, atol=get_default_atol(out), rtol=rtol[out.dtype]
|
||||
)
|
||||
|
||||
# Verify clamping is actually being applied: the clamped output should
|
||||
# differ from the unclamped SiluAndMul output when inputs are large.
|
||||
if alpha == 1.0 and beta == 0.0:
|
||||
unclamped_out = SiluAndMul.forward_native(x)
|
||||
assert not torch.equal(ref_out.float(), unclamped_out.float()), (
|
||||
"Input was not large enough to exercise the clamp; increase scale"
|
||||
)
|
||||
|
||||
# Verify gate clamping semantics with a controlled scalar case.
|
||||
# gate=large_val is clamped to limit first, then silu(limit) * 1.0.
|
||||
x_gate = torch.tensor(
|
||||
[[swiglu_limit * 20.0, 1.0]], dtype=torch.float32, device=device
|
||||
)
|
||||
out_gate = SiluAndMulWithClamp(swiglu_limit, compile_native=False)(x_gate)
|
||||
expected_gate = torch.nn.functional.silu(
|
||||
torch.tensor(swiglu_limit, dtype=torch.float32)
|
||||
).item()
|
||||
torch.testing.assert_close(
|
||||
out_gate,
|
||||
torch.tensor([[expected_gate]], dtype=torch.float32, device=device),
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
# Verify up clamping semantics: up >> limit gets clamped to limit.
|
||||
x_up = torch.tensor(
|
||||
[[1.0, swiglu_limit * 20.0]], dtype=torch.float32, device=device
|
||||
)
|
||||
out_up = SiluAndMulWithClamp(swiglu_limit, compile_native=False)(x_up)
|
||||
silu_1 = torch.nn.functional.silu(torch.tensor(1.0)).item()
|
||||
torch.testing.assert_close(
|
||||
out_up,
|
||||
torch.tensor([[silu_1 * swiglu_limit]], dtype=torch.float32, device=device),
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
# opcheck
|
||||
out_buf = torch.empty(x.shape[:-1] + (d,), dtype=dtype, device=device)
|
||||
opcheck(
|
||||
torch.ops._C.silu_and_mul_with_clamp,
|
||||
(out_buf, x, swiglu_limit, layer.alpha, layer.beta),
|
||||
)
|
||||
|
||||
|
||||
MOE_ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
|
||||
pytest.param(
|
||||
MoEActivation.SILU, ApplyMoEActivationConfig(clamp_limit=3.0), id="silu_clamp"
|
||||
),
|
||||
pytest.param(MoEActivation.GELU, ApplyMoEActivationConfig(), id="gelu"),
|
||||
pytest.param(MoEActivation.GELU_TANH, ApplyMoEActivationConfig(), id="gelu_tanh"),
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
id="situ",
|
||||
),
|
||||
pytest.param(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig(), id="swigluoai"),
|
||||
pytest.param(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
|
||||
id="swigluoai_uninterleave",
|
||||
),
|
||||
pytest.param(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig(), id="swiglustep"),
|
||||
pytest.param(
|
||||
MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig(), id="silu_no_mul"
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig(), id="gelu_no_mul"
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
ApplyMoEActivationConfig(),
|
||||
id="gelu_tanh_no_mul",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig(), id="relu2_no_mul"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("activation", "activation_config"), MOE_ACTIVATION_CASES)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_dispatch(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.bfloat16,
|
||||
mask_layout="batched_experts",
|
||||
d=513,
|
||||
max_num_tokens=7,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "activation_config", "mask_layout"),
|
||||
[
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
ApplyMoEActivationConfig(),
|
||||
"flat",
|
||||
id="flat",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=2.0,
|
||||
),
|
||||
"batched_experts",
|
||||
id="batched-experts",
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_grid_stride(
|
||||
default_vllm_config,
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
mask_layout: str,
|
||||
) -> None:
|
||||
_assert_masked_moe_activation(
|
||||
activation,
|
||||
activation_config,
|
||||
dtype=torch.half,
|
||||
mask_layout=mask_layout,
|
||||
d=513,
|
||||
max_num_tokens=67,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_masked_moe_activation_opcheck(default_vllm_config) -> None:
|
||||
device = CUDA_DEVICES[0]
|
||||
input = torch.randn(2, 3, 64, dtype=torch.half, device=device)
|
||||
output = torch.empty(2, 3, 32, dtype=torch.half, device=device)
|
||||
valid_token_counts = torch.tensor([1, 3], dtype=torch.int32, device=device)
|
||||
opcheck(
|
||||
torch.ops._C.masked_moe_activation,
|
||||
(output, input, valid_token_counts, "silu", 0.0, 1.0, 0.0, 1.0, -1.0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[
|
||||
(FastGELU, torch.ops._C.gelu_fast),
|
||||
(NewGELU, torch.ops._C.gelu_new),
|
||||
(QuickGELU, torch.ops._C.gelu_quick),
|
||||
(ReLUSquaredActivation, torch.ops._C.relu_squared),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("d", D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_activation(
|
||||
default_vllm_config,
|
||||
activation: type[torch.nn.Module],
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
x = torch.randn(num_tokens, d, dtype=dtype)
|
||||
layer = activation[0]()
|
||||
fn = activation[1]
|
||||
out = layer(x)
|
||||
ref_out = layer.forward_native(x)
|
||||
torch.testing.assert_close(
|
||||
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
|
||||
)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
opcheck(fn, (out, x))
|
||||
|
||||
|
||||
HUMMING_ACTIVATION_CASES = MOE_ACTIVATION_CASES + [
|
||||
pytest.param(MoEActivation.RELU2, ApplyMoEActivationConfig(), id="relu2"),
|
||||
]
|
||||
HUMMING_ACTIVATION_CASES += [
|
||||
pytest.param(
|
||||
MoEActivation.SITU,
|
||||
ApplyMoEActivationConfig(
|
||||
activation_situ_beta=1.5,
|
||||
activation_situ_linear_beta=linear_beta,
|
||||
),
|
||||
id=f"situ-linear-beta-{linear_beta}",
|
||||
)
|
||||
for linear_beta in (None, 0.0, -1.0)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Humming requires CUDA")
|
||||
@pytest.mark.parametrize(("activation", "activation_config"), HUMMING_ACTIVATION_CASES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize(("num_tokens", "d"), [(1, 512), (7, 768), (83, 512)])
|
||||
@torch.inference_mode()
|
||||
def test_humming_activation_matches_framework(
|
||||
activation: MoEActivation,
|
||||
activation_config: ApplyMoEActivationConfig,
|
||||
dtype: torch.dtype,
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
) -> None:
|
||||
"""Compare activation math/layouts without quantization or Hadamard error."""
|
||||
pytest.importorskip("humming")
|
||||
from humming.ops import process_input
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming.activation import (
|
||||
get_humming_activation,
|
||||
)
|
||||
|
||||
set_random_seed(0)
|
||||
width = d * 2 if activation.is_gated else d
|
||||
x = 4 * torch.randn(num_tokens, width, dtype=dtype, device=CUDA_DEVICES[0])
|
||||
# Exercise zero, saturation, and both sides of the clamp limits (3 and 7).
|
||||
edges = x.new_tensor([0, 0.001, 1, 2.99, 3, 3.01, 6.99, 7, 7.01, 8, 16])
|
||||
edges = torch.cat((-edges[1:].flip(0), edges))
|
||||
x[0] = edges[torch.arange(width, device=x.device) % edges.numel()]
|
||||
|
||||
actual, _, _ = process_input(
|
||||
x,
|
||||
quant_mode="none",
|
||||
hadamard_block_size=0,
|
||||
**get_humming_activation(activation, activation_config),
|
||||
)
|
||||
expected = torch.empty(num_tokens, d, dtype=dtype, device=x.device)
|
||||
if activation == MoEActivation.RELU2:
|
||||
# apply_moe_activation has only the non-gated ReLU2 variant.
|
||||
gate, up = x.float().chunk(2, dim=-1)
|
||||
activated_gate = torch.empty_like(gate)
|
||||
apply_moe_activation(MoEActivation.RELU2_NO_MUL, activated_gate, gate.clone())
|
||||
expected.copy_(activated_gate * up)
|
||||
else:
|
||||
# The framework's non-gated ReLU2 path modifies its input in place.
|
||||
apply_moe_activation(
|
||||
activation, expected, x.clone(), activation_config=activation_config
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual,
|
||||
expected,
|
||||
atol=get_default_atol(expected),
|
||||
rtol=get_default_rtol(expected),
|
||||
)
|
||||
@@ -1,283 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ApplyRotaryEmb CustomOp dispatch behavior.
|
||||
|
||||
This test ensures that RotaryEmbedding classes correctly call the appropriate
|
||||
ApplyRotaryEmb methods based on the calling context:
|
||||
|
||||
1. RotaryEmbedding.forward_native() -> ApplyRotaryEmb.forward_native()
|
||||
2. RotaryEmbedding.forward_cuda() -> ApplyRotaryEmb.forward() (auto-dispatch)
|
||||
3. RotaryEmbedding.forward_hip() -> ApplyRotaryEmb.forward() (auto-dispatch)
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
VllmConfig,
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class RotaryEmbeddingTestCase:
|
||||
"""Test case configuration for RotaryEmbedding dispatch tests."""
|
||||
|
||||
name: str
|
||||
rope_class: type
|
||||
rope_kwargs: dict
|
||||
method_name: str # forward_native, forward_cuda, forward
|
||||
positions_shape: tuple # (num_tokens,) or (3, num_tokens) or (4, num_tokens)
|
||||
expect_forward_native: bool # Should call ApplyRotaryEmb.forward_native()
|
||||
expect_forward: bool # Should call ApplyRotaryEmb.forward()
|
||||
|
||||
|
||||
def get_test_cases() -> list[RotaryEmbeddingTestCase]:
|
||||
"""Generate test cases for all RotaryEmbedding classes."""
|
||||
from vllm.model_executor.layers.rotary_embedding.ernie45_vl_rope import (
|
||||
Ernie4_5_VLRotaryEmbedding,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
|
||||
common_kwargs = {
|
||||
"head_size": 128,
|
||||
"rotary_dim": 128,
|
||||
"max_position_embeddings": 4096,
|
||||
"base": 10000,
|
||||
"is_neox_style": True,
|
||||
"dtype": torch.bfloat16,
|
||||
}
|
||||
|
||||
return [
|
||||
# MRotaryEmbedding tests
|
||||
RotaryEmbeddingTestCase(
|
||||
name="MRotaryEmbedding.forward_native",
|
||||
rope_class=MRotaryEmbedding,
|
||||
rope_kwargs={**common_kwargs, "mrope_section": [16, 24, 24]},
|
||||
method_name="forward_native",
|
||||
positions_shape=(3, 32), # 2D for multimodal
|
||||
expect_forward_native=True,
|
||||
expect_forward=False,
|
||||
),
|
||||
RotaryEmbeddingTestCase(
|
||||
name="MRotaryEmbedding.forward_cuda_1d",
|
||||
rope_class=MRotaryEmbedding,
|
||||
rope_kwargs={**common_kwargs, "mrope_section": [16, 24, 24]},
|
||||
method_name="forward_cuda",
|
||||
positions_shape=(32,), # 1D triggers apply_rotary_emb path
|
||||
expect_forward_native=False,
|
||||
expect_forward=True,
|
||||
),
|
||||
# Ernie4_5_VLRotaryEmbedding tests
|
||||
RotaryEmbeddingTestCase(
|
||||
name="Ernie4_5_VLRotaryEmbedding.forward_native",
|
||||
rope_class=Ernie4_5_VLRotaryEmbedding,
|
||||
rope_kwargs={**common_kwargs, "mrope_section": [22, 22, 20]},
|
||||
method_name="forward_native",
|
||||
positions_shape=(3, 32), # 2D for multimodal
|
||||
expect_forward_native=True,
|
||||
expect_forward=False,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def run_dispatch_test(
|
||||
test_case: RotaryEmbeddingTestCase,
|
||||
device: str,
|
||||
):
|
||||
"""Run a dispatch test for a RotaryEmbedding class."""
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"])
|
||||
)
|
||||
get_cached_compilation_config.cache_clear()
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
rope = test_case.rope_class(**test_case.rope_kwargs).to(device=device)
|
||||
|
||||
apply_rotary_emb = rope.apply_rotary_emb
|
||||
|
||||
# Verify custom op is enabled
|
||||
if test_case.expect_forward_native:
|
||||
assert (
|
||||
apply_rotary_emb._forward_method != apply_rotary_emb.forward_native
|
||||
), "Test setup error: ApplyRotaryEmb custom op should be enabled"
|
||||
|
||||
# Setup call tracking
|
||||
call_tracker = {"forward_native_called": False, "forward_called": False}
|
||||
original_forward_native = apply_rotary_emb.forward_native
|
||||
original_forward = apply_rotary_emb.forward
|
||||
|
||||
def tracked_forward_native(*args, **kwargs):
|
||||
call_tracker["forward_native_called"] = True
|
||||
return original_forward_native(*args, **kwargs)
|
||||
|
||||
def tracked_forward(*args, **kwargs):
|
||||
call_tracker["forward_called"] = True
|
||||
return original_forward(*args, **kwargs)
|
||||
|
||||
apply_rotary_emb.forward_native = tracked_forward_native
|
||||
apply_rotary_emb.forward = tracked_forward
|
||||
|
||||
try:
|
||||
num_tokens = test_case.positions_shape[-1]
|
||||
num_q_heads = 8
|
||||
num_kv_heads = 2
|
||||
head_size = test_case.rope_kwargs["head_size"]
|
||||
max_position = test_case.rope_kwargs["max_position_embeddings"]
|
||||
|
||||
positions = torch.randint(
|
||||
0, max_position // 4, test_case.positions_shape, device=device
|
||||
)
|
||||
query = torch.randn(
|
||||
num_tokens, num_q_heads * head_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
key = torch.randn(
|
||||
num_tokens,
|
||||
num_kv_heads * head_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Call the method under test
|
||||
method = getattr(rope, test_case.method_name)
|
||||
method(positions, query.clone(), key.clone())
|
||||
|
||||
# Verify expectations
|
||||
if test_case.expect_forward_native:
|
||||
assert call_tracker["forward_native_called"], (
|
||||
f"{test_case.name} should call ApplyRotaryEmb.forward_native()"
|
||||
)
|
||||
if not test_case.expect_forward:
|
||||
assert not call_tracker["forward_called"], (
|
||||
f"{test_case.name} should NOT call ApplyRotaryEmb.forward(). "
|
||||
"Bug: when +apply_rotary_emb is enabled, forward_native() "
|
||||
"incorrectly dispatches to CUDA/HIP kernels."
|
||||
)
|
||||
if test_case.expect_forward:
|
||||
assert call_tracker["forward_called"], (
|
||||
f"{test_case.name} should call ApplyRotaryEmb.forward()"
|
||||
)
|
||||
finally:
|
||||
apply_rotary_emb.forward_native = original_forward_native
|
||||
apply_rotary_emb.forward = original_forward
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
)
|
||||
@pytest.mark.parametrize("test_case", get_test_cases(), ids=lambda tc: tc.name)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_rotary_embedding_dispatch(
|
||||
test_case: RotaryEmbeddingTestCase,
|
||||
device: str,
|
||||
):
|
||||
"""Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
|
||||
|
||||
- forward_native methods should call ApplyRotaryEmb.forward_native()
|
||||
- forward_cuda/forward methods should call ApplyRotaryEmb.forward()
|
||||
"""
|
||||
run_dispatch_test(test_case, device)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_xpu(), reason="XPU only test.")
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,num_heads,head_size,rot_dim,is_neox_style",
|
||||
[
|
||||
(128, 32, 128, 128, True), # typical text shape
|
||||
(4096, 16, 80, 80, False), # GPT-J style (interleaved)
|
||||
(4096, 32, 128, 64, True), # partial rotary (rot_dim < head_size)
|
||||
],
|
||||
ids=["text", "gptj_interleaved", "partial_rotary"],
|
||||
)
|
||||
def test_apply_rotary_emb_xpu_matches_native(
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
rot_dim: int,
|
||||
is_neox_style: bool,
|
||||
):
|
||||
"""ApplyRotaryEmb.forward_xpu (vllm_xpu_kernels SYCL op) must match
|
||||
forward_native (the CustomOp default fallback) exactly."""
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["all", "+apply_rotary_emb"])
|
||||
)
|
||||
get_cached_compilation_config.cache_clear()
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
op = ApplyRotaryEmb(enforce_enable=True, is_neox_style=is_neox_style)
|
||||
|
||||
x = torch.randn(
|
||||
num_tokens, num_heads, head_size, device="xpu", dtype=torch.bfloat16
|
||||
)
|
||||
x = x[..., :rot_dim].contiguous()
|
||||
cos = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
|
||||
sin = torch.randn(num_tokens, rot_dim // 2, device="xpu", dtype=torch.bfloat16)
|
||||
|
||||
expected = op.forward_native(x, cos, sin)
|
||||
actual = op.forward_xpu(x, cos, sin)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm only test.")
|
||||
@pytest.mark.parametrize("num_tokens", [257, 2304])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_packed_qk_rope_correctness(
|
||||
num_tokens: int, dtype: torch.dtype, default_vllm_config
|
||||
):
|
||||
"""packed_qk_rope_ must match the per-tensor ApplyRotaryEmb path with
|
||||
enable_fp32_compute=True to within one ULP, and must leave the V slice
|
||||
untouched.
|
||||
|
||||
Both sides compute in fp32 on identical inputs. The only gap is which
|
||||
product the backend keeps exact inside the o1 multiply-add, and that
|
||||
follows the register layout, so it is decided by codegen rather than by
|
||||
the source and holds neither across backends nor across Triton versions.
|
||||
"""
|
||||
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
|
||||
from vllm.model_executor.layers.rotary_embedding.packed_qk_rope import (
|
||||
packed_qk_rope_,
|
||||
)
|
||||
|
||||
num_heads, head_dim = 12, 128
|
||||
rng = torch.Generator(device="cuda").manual_seed(0)
|
||||
|
||||
angles = (
|
||||
torch.rand(num_tokens, head_dim // 2, device="cuda", generator=rng) * torch.pi
|
||||
)
|
||||
freqs_cis = torch.polar(torch.ones_like(angles), angles)
|
||||
cos = freqs_cis.real.contiguous()
|
||||
sin = freqs_cis.imag.contiguous()
|
||||
|
||||
xqkv = torch.randn(
|
||||
num_tokens, 3, num_heads, head_dim, dtype=dtype, device="cuda", generator=rng
|
||||
)
|
||||
xqkv_ref = xqkv.clone()
|
||||
xq, xk, xv = torch.unbind(xqkv_ref, dim=-3)
|
||||
|
||||
op = ApplyRotaryEmb(
|
||||
enforce_enable=True, is_neox_style=False, enable_fp32_compute=True
|
||||
)
|
||||
# op() dispatches to forward_cuda / forward_hip per platform.
|
||||
xq_ref = op(xq, cos, sin)
|
||||
xk_ref = op(xk, cos, sin)
|
||||
|
||||
packed_qk_rope_(xqkv, freqs_cis)
|
||||
|
||||
# One ULP at the largest magnitude present, not relative: o1 can cancel,
|
||||
# so the error scales with the inputs to the sum, not with the result.
|
||||
eps = torch.finfo(dtype).eps
|
||||
q_atol = eps * xq_ref.abs().max().item()
|
||||
k_atol = eps * xk_ref.abs().max().item()
|
||||
torch.testing.assert_close(xqkv[:, 0], xq_ref, atol=q_atol, rtol=0)
|
||||
torch.testing.assert_close(xqkv[:, 1], xk_ref, atol=k_atol, rtol=0)
|
||||
# V is never read or written by the kernel, so it stays exact.
|
||||
torch.testing.assert_close(xqkv[:, 2], xv, atol=0, rtol=0)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the batched-weight RMS norm kernel (vllm._custom_ops.rms_norm).
|
||||
|
||||
``rms_norm`` can use the outermost input batch index to select the corresponding
|
||||
weight row. The result must match that of looping ``rms_norm`` over that dimension.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="rms_norm requires a CUDA/ROCm device",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(28, 17, 128), # 3D: [num_rows, tokens, hidden]
|
||||
(1, 5, 2, 128), # 4D: single row (edge case)
|
||||
(28, 13, 8, 128), # 4D: [L, num_ctx, nkv, hd] (DFlash K-norm)
|
||||
(6, 3, 4, 769), # 4D: non-power-of-two hidden size
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16, torch.float])
|
||||
@pytest.mark.parametrize("seed", [42])
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm_matches_loop(
|
||||
shape: tuple[int, ...], dtype: torch.dtype, seed: int
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
num_rows, hidden = shape[0], shape[-1]
|
||||
eps = 1e-6
|
||||
|
||||
x = torch.randn(*shape, dtype=dtype) * 0.1
|
||||
# Distinct weight per row so that a wrong row index would be caught.
|
||||
weight = torch.randn(num_rows, hidden, dtype=dtype) * 0.1 + 1.0
|
||||
|
||||
# Reference batched-weight rms norm.
|
||||
out_ref = torch.empty_like(x)
|
||||
for i in range(x.shape[0]):
|
||||
ops.rms_norm(out_ref[i], x[i], weight[i], eps)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
ops.rms_norm(out, x, weight, eps)
|
||||
|
||||
# Expect bitwise-identical results.
|
||||
torch.testing.assert_close(out, out_ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm_validates_shapes() -> None:
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
x = torch.randn(4, 8, 128, dtype=torch.float)
|
||||
out = torch.empty_like(x)
|
||||
# Expect num rows mismatch.
|
||||
with pytest.raises(RuntimeError):
|
||||
ops.rms_norm(out, x, torch.randn(3, 128), 1e-6)
|
||||
# Expect hidden size mismatch.
|
||||
with pytest.raises(RuntimeError):
|
||||
ops.rms_norm(out, x, torch.randn(4, 64), 1e-6)
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.layers.activation import (
|
||||
GELU,
|
||||
FastGELU,
|
||||
GeluAndMul,
|
||||
NewGELU,
|
||||
QuickGELU,
|
||||
SiluAndMul,
|
||||
)
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float32]
|
||||
NUM_TOKENS = [7, 83]
|
||||
D = [512, 2048]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation_cls", "fn"),
|
||||
[
|
||||
(SiluAndMul, torch.ops._C.silu_and_mul),
|
||||
(GeluAndMul, torch.ops._C.gelu_and_mul),
|
||||
(GeluAndMul, torch.ops._C.gelu_tanh_and_mul),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("d", D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_cpu_act_and_mul(
|
||||
default_vllm_config,
|
||||
activation_cls: type[torch.nn.Module],
|
||||
fn: object,
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
|
||||
|
||||
layer = activation_cls()
|
||||
out = layer(x)
|
||||
ref_out = layer.forward_native(x)
|
||||
|
||||
torch.testing.assert_close(
|
||||
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
|
||||
)
|
||||
|
||||
output_shape = x.shape[:-1] + (x.shape[-1] // 2,)
|
||||
raw_out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
opcheck(fn, (raw_out, x))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation_cls", "fn", "op_args"),
|
||||
[
|
||||
(NewGELU, torch.ops._C.gelu_new, ()),
|
||||
(FastGELU, torch.ops._C.gelu_fast, ()),
|
||||
(QuickGELU, torch.ops._C.gelu_quick, ()),
|
||||
pytest.param(
|
||||
GELU,
|
||||
getattr(torch.ops._C, "activation_lut_bf16", None),
|
||||
("gelu",),
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.get_cpu_architecture() != CpuArchEnum.ARM,
|
||||
reason="activation_lut_bf16 is only built on Arm CPU",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("d", D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_cpu_unary_activation(
|
||||
default_vllm_config,
|
||||
activation_cls: type[torch.nn.Module],
|
||||
fn: object,
|
||||
op_args: tuple[str, ...],
|
||||
num_tokens: int,
|
||||
d: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
x = torch.randn(num_tokens, d, dtype=dtype)
|
||||
layer = activation_cls()
|
||||
out = layer(x)
|
||||
ref_out = layer.forward_native(x)
|
||||
torch.testing.assert_close(
|
||||
out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
|
||||
)
|
||||
# gelu with activation_lut_bf16 only makes sense for BF16
|
||||
if not (activation_cls is GELU and dtype != torch.bfloat16):
|
||||
raw_out = torch.empty_like(x)
|
||||
opcheck(fn, (raw_out, x, *op_args))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode()
|
||||
def test_cpu_gelu_tanh_and_mul(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
gate = torch.tensor(
|
||||
[
|
||||
[
|
||||
-12.0,
|
||||
-10.0,
|
||||
-9.01,
|
||||
-5.0,
|
||||
-2.0,
|
||||
-1.0,
|
||||
-0.0,
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
2.0,
|
||||
5.0,
|
||||
9.01,
|
||||
10.0,
|
||||
12.0,
|
||||
11.0,
|
||||
],
|
||||
[
|
||||
-7.5,
|
||||
-4.5,
|
||||
-3.0,
|
||||
-1.5,
|
||||
-0.75,
|
||||
-0.25,
|
||||
0.25,
|
||||
0.75,
|
||||
1.5,
|
||||
3.0,
|
||||
4.5,
|
||||
7.5,
|
||||
-11.0,
|
||||
11.0,
|
||||
8.75,
|
||||
-8.75,
|
||||
],
|
||||
],
|
||||
dtype=dtype,
|
||||
)
|
||||
val = torch.tensor(
|
||||
[
|
||||
[
|
||||
0.25,
|
||||
-0.5,
|
||||
0.75,
|
||||
-1.0,
|
||||
1.25,
|
||||
-1.5,
|
||||
1.75,
|
||||
-2.0,
|
||||
2.25,
|
||||
-2.5,
|
||||
2.75,
|
||||
-3.0,
|
||||
3.25,
|
||||
-3.5,
|
||||
3.75,
|
||||
-4.0,
|
||||
],
|
||||
[
|
||||
-0.4,
|
||||
0.6,
|
||||
-0.8,
|
||||
1.0,
|
||||
-1.2,
|
||||
1.4,
|
||||
-1.6,
|
||||
1.8,
|
||||
-2.0,
|
||||
2.2,
|
||||
-2.4,
|
||||
2.6,
|
||||
-2.8,
|
||||
3.0,
|
||||
-3.2,
|
||||
3.4,
|
||||
],
|
||||
],
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
x = torch.cat((val, gate), dim=-1).contiguous()
|
||||
kernel_out = torch.empty_like(val)
|
||||
torch.ops._C.gelu_tanh_and_mul(kernel_out, x)
|
||||
|
||||
torch_ref = torch.nn.functional.gelu(val, approximate="tanh") * gate
|
||||
|
||||
atol = get_default_atol(kernel_out)
|
||||
rtol = get_default_rtol(kernel_out)
|
||||
torch.testing.assert_close(kernel_out, torch_ref, atol=atol, rtol=rtol)
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the manual AllReduce + GemmaRMSNorm fusion used by MiniMax M3.
|
||||
|
||||
``fused_allreduce_gemma_rms_norm`` must match the unfused model path, i.e.
|
||||
``GemmaRMSNorm(all_reduce(partial), residual)``, both on the flashinfer fast
|
||||
path (TP>1 with flashinfer + NVSwitch) and on the eager fallback (TP==1, or when
|
||||
flashinfer is unavailable / the GPU has no NVSwitch).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.multiprocessing import spawn
|
||||
|
||||
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@ensure_current_vllm_config()
|
||||
def _worker_fused_ar_norm(
|
||||
local_rank,
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
):
|
||||
"""Per-rank worker: compare the fused helper vs all_reduce + GemmaRMSNorm."""
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
|
||||
# Norm weights are identical across ranks (replicated GemmaRMSNorm).
|
||||
set_random_seed(seed)
|
||||
norm = GemmaRMSNorm(hidden_size, eps=eps).cuda().to(dtype)
|
||||
with torch.no_grad():
|
||||
norm.weight.normal_(mean=0.0, std=0.1)
|
||||
|
||||
# Residual is shared across ranks; the partial o_proj output differs per rank
|
||||
# (each rank holds a partial sum that all_reduce combines).
|
||||
torch.manual_seed(seed + 7)
|
||||
residual = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
torch.manual_seed(seed + 1000 + local_rank)
|
||||
partial = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
|
||||
# Reference: the unfused model path.
|
||||
reduced = tensor_model_parallel_all_reduce(partial.clone())
|
||||
ref_out, ref_res = norm(reduced, residual.clone())
|
||||
|
||||
# Fused helper (flashinfer fast path when available, else fallback).
|
||||
out, res = fused_allreduce_gemma_rms_norm(partial.clone(), residual.clone(), norm)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(res, ref_res, atol=2e-2, rtol=2e-2)
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA required",
|
||||
)
|
||||
# world_size=1 exercises the TP==1 identity branch on a single GPU; >1 exercises
|
||||
# the all_reduce + GemmaRMSNorm equivalence (flashinfer kernel or fallback).
|
||||
@pytest.mark.parametrize("world_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
|
||||
@pytest.mark.parametrize("hidden_size", [2048, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.parametrize("seed", [42])
|
||||
def test_fused_allreduce_gemma_rms_norm(
|
||||
world_size,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
eps,
|
||||
seed,
|
||||
):
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus < world_size:
|
||||
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
|
||||
port = str(get_open_port())
|
||||
spawn(
|
||||
_worker_fused_ar_norm,
|
||||
args=(
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
),
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
)
|
||||
@@ -1,148 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
IS_NEOX = [True, False]
|
||||
EPS_VALUES = [1e-5, 1e-6]
|
||||
SEEDS = [13]
|
||||
PARTIAL_ROPE = [True, False]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
def _apply_qk_norm_rope(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_norm: RMSNorm,
|
||||
k_norm: RMSNorm,
|
||||
rope: RotaryEmbedding,
|
||||
num_heads_q: int,
|
||||
num_heads_kv: int,
|
||||
head_dim: int,
|
||||
) -> torch.Tensor:
|
||||
q_size = num_heads_q * head_dim
|
||||
kv_size = num_heads_kv * head_dim
|
||||
|
||||
q, k, v = qkv.split([q_size, kv_size, kv_size], dim=-1)
|
||||
|
||||
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // head_dim, head_dim)
|
||||
q_by_head = q_norm.forward_native(q_by_head)
|
||||
assert isinstance(q_by_head, torch.Tensor)
|
||||
q = q_by_head.view(q.shape)
|
||||
|
||||
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // head_dim, head_dim)
|
||||
k_by_head = k_norm.forward_native(k_by_head)
|
||||
assert isinstance(k_by_head, torch.Tensor)
|
||||
k = k_by_head.view(k.shape)
|
||||
|
||||
q, k = rope.forward_native(positions, q, k)
|
||||
return torch.cat([q, k, v], dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fused_qk_norm_rope custom op requires cuda and rocm platform",
|
||||
)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("is_neox", IS_NEOX)
|
||||
@pytest.mark.parametrize("eps", EPS_VALUES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
|
||||
@torch.inference_mode()
|
||||
def test_fused_qk_norm_rope_matches_reference(
|
||||
default_vllm_config,
|
||||
device: str,
|
||||
dtype: torch.dtype,
|
||||
is_neox: bool,
|
||||
eps: float,
|
||||
seed: int,
|
||||
rotary_ratio: float,
|
||||
):
|
||||
torch.set_default_device(device)
|
||||
set_random_seed(seed)
|
||||
num_heads, num_kv_heads, head_dim = 16, 4, 128
|
||||
num_tokens = 4
|
||||
|
||||
total_dim = (num_heads + 2 * num_kv_heads) * head_dim
|
||||
qkv_base = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
|
||||
qkv_fused = qkv_base.clone()
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=device)
|
||||
|
||||
q_norm = RMSNorm(head_dim, eps=eps).to(device=device, dtype=dtype)
|
||||
k_norm = RMSNorm(head_dim, eps=eps).to(device=device, dtype=dtype)
|
||||
q_norm.weight.data.normal_(mean=1.0, std=0.1)
|
||||
k_norm.weight.data.normal_(mean=1.0, std=0.1)
|
||||
q_weight = q_norm.weight.data
|
||||
k_weight = k_norm.weight.data
|
||||
rotary_dim = int(head_dim * rotary_ratio)
|
||||
rope = RotaryEmbedding(
|
||||
head_size=head_dim,
|
||||
rotary_dim=rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000.0,
|
||||
is_neox_style=is_neox,
|
||||
dtype=dtype,
|
||||
).to(device)
|
||||
|
||||
ref_result = _apply_qk_norm_rope(
|
||||
qkv=qkv_base,
|
||||
positions=positions,
|
||||
q_norm=q_norm,
|
||||
k_norm=k_norm,
|
||||
rope=rope,
|
||||
num_heads_q=num_heads,
|
||||
num_heads_kv=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.fused_qk_norm_rope,
|
||||
(
|
||||
qkv_fused.clone(),
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rope.cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
),
|
||||
)
|
||||
|
||||
torch.ops._C.fused_qk_norm_rope(
|
||||
qkv_fused,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rope.cos_sin_cache,
|
||||
is_neox,
|
||||
positions.view(-1),
|
||||
)
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
torch.testing.assert_close(
|
||||
qkv_fused,
|
||||
ref_result,
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
@@ -1,426 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.utils import fp8_allclose, fp8_ulp_distance, opcheck
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
ON_GFX950 = False
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
ON_GFX950 = on_gfx950()
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
QUANT_DTYPES = [torch.int8, current_platform.fp8_dtype()]
|
||||
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
|
||||
# Avoid combinatorial explosion with full Cartesian product
|
||||
NUM_TOKENS_HIDDEN_SIZES = [
|
||||
*[(1, i) for i in [1, 64, 128, *VEC_HIDDEN_SIZES, 5120, 5137]],
|
||||
*[(2048, i) for i in [1, 64, *VEC_HIDDEN_SIZES, 5137]],
|
||||
*[(4096, i) for i in [1, 64, 5137]],
|
||||
]
|
||||
|
||||
ADD_RESIDUAL = [False, True]
|
||||
SCALE_UBS = [True, False]
|
||||
GROUP_SIZES = [[1, 64], [1, 128]]
|
||||
TMA_ALIGNMENTS = [0, 4]
|
||||
SEEDS = [0]
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def _is_valid_config(
|
||||
hidden_size: int,
|
||||
has_scale_ub: bool,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: list[int] | None,
|
||||
tma_alignment: int,
|
||||
) -> bool:
|
||||
if group_size is not None and hidden_size % group_size[1] != 0:
|
||||
return False
|
||||
if group_size is not None and has_scale_ub:
|
||||
return False
|
||||
if (
|
||||
group_size is None or quant_dtype != current_platform.fp8_dtype()
|
||||
) and tma_alignment != 0:
|
||||
return False
|
||||
if (
|
||||
group_size is not None
|
||||
and tma_alignment != 0
|
||||
and hidden_size // group_size[1] % tma_alignment == 0
|
||||
):
|
||||
return False
|
||||
return not (has_scale_ub and quant_dtype != current_platform.fp8_dtype())
|
||||
|
||||
|
||||
def _config_id(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
has_scale_ub: bool,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: list[int] | None,
|
||||
tma_alignment: int,
|
||||
) -> str:
|
||||
quant = str(quant_dtype).removeprefix("torch.")
|
||||
group = "per-token" if group_size is None else f"group-{group_size[1]}"
|
||||
return (
|
||||
f"{num_tokens}x{hidden_size}-{quant}-{group}-"
|
||||
f"tma-{tma_alignment}-scale-ub-{has_scale_ub}"
|
||||
)
|
||||
|
||||
|
||||
# Filter unsupported combinations during collection. Letting each case call
|
||||
# pytest.skip still runs the global per-test teardown and dominates this suite.
|
||||
RMS_NORM_CONFIGS = [
|
||||
pytest.param(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
has_scale_ub,
|
||||
quant_dtype,
|
||||
group_size,
|
||||
tma_alignment,
|
||||
id=_config_id(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
has_scale_ub,
|
||||
quant_dtype,
|
||||
group_size,
|
||||
tma_alignment,
|
||||
),
|
||||
)
|
||||
for (
|
||||
(num_tokens, hidden_size),
|
||||
has_scale_ub,
|
||||
quant_dtype,
|
||||
(group_size, tma_alignment),
|
||||
) in itertools.product(
|
||||
NUM_TOKENS_HIDDEN_SIZES,
|
||||
SCALE_UBS,
|
||||
QUANT_DTYPES,
|
||||
[(None, 0), *itertools.product(GROUP_SIZES, TMA_ALIGNMENTS)],
|
||||
)
|
||||
if _is_valid_config(
|
||||
hidden_size,
|
||||
has_scale_ub,
|
||||
quant_dtype,
|
||||
group_size,
|
||||
tma_alignment,
|
||||
)
|
||||
]
|
||||
|
||||
## Helpers
|
||||
|
||||
|
||||
def as_float32_tensor(x: float | torch.Tensor) -> torch.Tensor:
|
||||
return torch.as_tensor(x, dtype=torch.float32, device="cuda")
|
||||
|
||||
|
||||
def ref_rms_norm(
|
||||
rms_norm_layer: RMSNorm, x: torch.Tensor, residual: torch.Tensor | None
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
if residual is not None:
|
||||
residual = residual.clone()
|
||||
out, residual = rms_norm_layer.forward_native(x, residual)
|
||||
else:
|
||||
out = rms_norm_layer.forward_native(x)
|
||||
|
||||
return out, residual
|
||||
|
||||
|
||||
def ref_dynamic_per_token_or_block_quant(
|
||||
rms_norm_layer: RMSNorm,
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
residual: torch.Tensor | None,
|
||||
scale_ub: torch.Tensor | None,
|
||||
group_size: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
if scale_ub is not None:
|
||||
assert quant_dtype == current_platform.fp8_dtype()
|
||||
|
||||
# Norm
|
||||
torch_out, residual = ref_rms_norm(rms_norm_layer, x, residual)
|
||||
|
||||
# Quant
|
||||
if group_size is not None:
|
||||
if quant_dtype == current_platform.fp8_dtype():
|
||||
torch_out, scales = per_token_group_quant_fp8(
|
||||
torch_out, group_size=group_size[1], use_ue8m0=False
|
||||
)
|
||||
else:
|
||||
assert quant_dtype == torch.int8
|
||||
torch_out, scales = per_token_group_quant_int8(
|
||||
torch_out, group_size=group_size[1]
|
||||
)
|
||||
else:
|
||||
if quant_dtype == current_platform.fp8_dtype():
|
||||
torch_out, scales = ops.scaled_fp8_quant(
|
||||
torch_out, scale_ub=scale_ub, use_per_token_if_dynamic=True
|
||||
)
|
||||
else:
|
||||
assert quant_dtype == torch.int8
|
||||
torch_out, scales, _ = ops.scaled_int8_quant(torch_out)
|
||||
|
||||
return torch_out, scales, residual
|
||||
|
||||
|
||||
def ref_impl(
|
||||
rms_norm_layer: RMSNorm,
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
residual: torch.Tensor | None,
|
||||
scale_ub: torch.Tensor | None,
|
||||
group_size: list[int] | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
return ref_dynamic_per_token_or_block_quant(
|
||||
rms_norm_layer, x, quant_dtype, residual, scale_ub, group_size
|
||||
)
|
||||
|
||||
|
||||
def ops_dynamic_per_token_or_block_quant(
|
||||
weight: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
residual: torch.Tensor | None,
|
||||
scale_ub: torch.Tensor | None,
|
||||
group_size: list[int] | None,
|
||||
tma_alignment: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
if residual is not None:
|
||||
residual = residual.clone()
|
||||
if group_size is not None:
|
||||
out, scales = ops.rms_norm_per_block_quant(
|
||||
x,
|
||||
weight,
|
||||
EPS,
|
||||
quant_dtype,
|
||||
group_size,
|
||||
scale_ub,
|
||||
residual,
|
||||
True,
|
||||
tma_alignment,
|
||||
)
|
||||
scales = scales.contiguous()
|
||||
else:
|
||||
out, scales = ops.rms_norm_dynamic_per_token_quant(
|
||||
x, weight, EPS, quant_dtype, scale_ub, residual
|
||||
)
|
||||
return out, scales, residual
|
||||
|
||||
|
||||
def ops_impl(
|
||||
weight: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
residual: torch.Tensor | None,
|
||||
scale_ub: torch.Tensor | None,
|
||||
group_size: list[int] | None,
|
||||
tma_alignment: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
return ops_dynamic_per_token_or_block_quant(
|
||||
weight, x, quant_dtype, residual, scale_ub, group_size, tma_alignment
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"hidden_size",
|
||||
"has_scale_ub",
|
||||
"quant_dtype",
|
||||
"group_size",
|
||||
"tma_alignment",
|
||||
),
|
||||
RMS_NORM_CONFIGS,
|
||||
)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("strided_input", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
has_scale_ub: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: list[int] | None,
|
||||
tma_alignment: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
strided_input: bool,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
torch.accelerator.set_device_index(device)
|
||||
|
||||
layer = RMSNorm(hidden_size, EPS).to(dtype=dtype)
|
||||
|
||||
# Make weights
|
||||
layer.weight.data.normal_(mean=1.0, std=0.1)
|
||||
|
||||
# Make inputs: use a wider tensor and slice to create a non-contiguous
|
||||
# (strided) input when strided_input=True. The last dimension stride
|
||||
# remains 1, which the kernel requires.
|
||||
scale = 1 / (hidden_size)
|
||||
last_dim = 2 * hidden_size if strided_input else hidden_size
|
||||
x = torch.randn(num_tokens, last_dim, dtype=dtype) * scale
|
||||
x = x[:, :hidden_size]
|
||||
|
||||
# dim 1 gets special-cased
|
||||
x_is_strided = strided_input and num_tokens != 1
|
||||
# check that the input is strided iff we expect it to be
|
||||
assert x.is_contiguous() != x_is_strided
|
||||
|
||||
# Residual must still be contiguous
|
||||
residual = (
|
||||
torch.randn(num_tokens, hidden_size, dtype=dtype) * scale
|
||||
if add_residual
|
||||
else None
|
||||
)
|
||||
if has_scale_ub:
|
||||
rms_x, _ = ref_rms_norm(layer, x, residual)
|
||||
scale_ub = torch.mean(rms_x).to(dtype=torch.float32, device="cuda")
|
||||
else:
|
||||
scale_ub = None
|
||||
|
||||
ref_out, ref_scales, ref_residual = ref_impl(
|
||||
layer, x, quant_dtype, residual, scale_ub, group_size
|
||||
)
|
||||
ops_out, ops_scales, ops_residual = ops_impl(
|
||||
layer.weight, x, quant_dtype, residual, scale_ub, group_size, tma_alignment
|
||||
)
|
||||
|
||||
assert ref_out.dtype == quant_dtype
|
||||
assert ops_out.dtype == quant_dtype
|
||||
|
||||
# Per-block bf16 scales: allow a small relative tolerance for a few groups
|
||||
# whose abs-max flips by one ULP between the fused and reference paths. The
|
||||
# per-token and fp32 paths stay strict.
|
||||
# The same one-ULP group-scale flip also occurs on CUDA (H200), so extend
|
||||
# the block relaxation there too — the fused groupwise reduction rounds
|
||||
# differently than the reference for isolated groups.
|
||||
relax_block = (
|
||||
group_size is not None
|
||||
and dtype == torch.bfloat16
|
||||
and (current_platform.is_rocm() or current_platform.is_cuda())
|
||||
)
|
||||
use_gfx950_fp8_allclose = (
|
||||
current_platform.is_rocm()
|
||||
and ON_GFX950
|
||||
and group_size is None
|
||||
and dtype == torch.bfloat16
|
||||
and quant_dtype == current_platform.fp8_dtype()
|
||||
)
|
||||
allow_cuda_fp8_rounding_outliers = (
|
||||
current_platform.is_cuda()
|
||||
and group_size is None
|
||||
and dtype == torch.bfloat16
|
||||
and quant_dtype == current_platform.fp8_dtype()
|
||||
)
|
||||
|
||||
def scales_close(rtol: float, atol: float) -> bool:
|
||||
if torch.allclose(ref_scales, ops_scales, rtol=rtol, atol=atol):
|
||||
return True
|
||||
return relax_block and torch.allclose(
|
||||
ref_scales, ops_scales, rtol=1e-2, atol=atol
|
||||
)
|
||||
|
||||
if quant_dtype == torch.int8:
|
||||
assert scales_close(rtol=1e-5, atol=1e-6)
|
||||
# big atol to account for round-off errors.
|
||||
assert torch.allclose(ref_out, ops_out, atol=1)
|
||||
else:
|
||||
assert scales_close(rtol=1e-5, atol=1e-8)
|
||||
a = ref_out.to(dtype=torch.float32)
|
||||
b = ops_out.to(dtype=torch.float32)
|
||||
ok = torch.allclose(a, b, atol=1e-6)
|
||||
if not ok:
|
||||
if relax_block:
|
||||
# ULP-flipped group scale can cross an E4M3 tie; tolerate a
|
||||
# bounded count of isolated fp8 outliers.
|
||||
ulp = fp8_ulp_distance(ref_out, ops_out)
|
||||
max_outliers = ulp.numel() // 100_000 + 8
|
||||
ok = int((ulp > 0).sum().item()) <= max_outliers
|
||||
elif use_gfx950_fp8_allclose:
|
||||
# Valid gfx950 reduction trees can straddle an E4M3 boundary.
|
||||
ok = fp8_allclose(ops_out, ref_out, rtol=0.125, atol=2e-3)
|
||||
ok = ok and int(fp8_ulp_distance(ops_out, ref_out).max()) <= 1
|
||||
elif allow_cuda_fp8_rounding_outliers:
|
||||
# A valid BF16 reduction can cross an E4M3 boundary for
|
||||
# isolated values.
|
||||
ulp = fp8_ulp_distance(ref_out, ops_out)
|
||||
max_outliers = ulp.numel() // 100_000 + 8
|
||||
ok = int(ulp.max()) <= 1
|
||||
ok = ok and int((ulp > 0).sum().item()) <= max_outliers
|
||||
else:
|
||||
# Compare dequantized values with relaxed tolerance.
|
||||
if group_size is None:
|
||||
a_deq = a * ref_scales.view(-1, 1)
|
||||
b_deq = b * ops_scales.view(-1, 1)
|
||||
else:
|
||||
a_deq = a * ref_scales.repeat_interleave(group_size[1], dim=1)
|
||||
b_deq = b * ops_scales.repeat_interleave(group_size[1], dim=1)
|
||||
# NOTE: It is possible that some future test cases trigger this
|
||||
# max diff due to precision issues. If such an error is
|
||||
# encountered, it's recommended to inspect the differences between
|
||||
# all corresponding elements from each tensor (e.g. by looping over
|
||||
# them) and checking how many the max diff error shows up on (just
|
||||
# a few bad elements should still be considered acceptable).
|
||||
ok = torch.allclose(a_deq, b_deq, rtol=5e-2, atol=5e-2)
|
||||
assert ok
|
||||
if add_residual:
|
||||
assert torch.allclose(ref_residual, ops_residual)
|
||||
|
||||
output = torch.empty(x.shape, dtype=quant_dtype, device=x.device)
|
||||
if group_size is None:
|
||||
scales = torch.empty(
|
||||
(x.numel() // x.shape[-1], 1), device=x.device, dtype=torch.float32
|
||||
)
|
||||
opcheck(
|
||||
torch.ops._C.rms_norm_dynamic_per_token_quant,
|
||||
(output, x, layer.weight, scales, 1e-5, scale_ub, residual),
|
||||
)
|
||||
else:
|
||||
assert hidden_size % group_size[1] == 0
|
||||
num_groups = hidden_size // group_size[1]
|
||||
scales = torch.empty(
|
||||
(num_groups, num_tokens),
|
||||
device=x.device,
|
||||
dtype=torch.float32,
|
||||
).transpose(0, 1)
|
||||
opcheck(
|
||||
torch.ops._C.rms_norm_per_block_quant,
|
||||
(
|
||||
output,
|
||||
x,
|
||||
layer.weight,
|
||||
scales,
|
||||
1e-5,
|
||||
scale_ub,
|
||||
residual,
|
||||
group_size[1],
|
||||
True, # is_scale_transposed
|
||||
),
|
||||
)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
|
||||
matching the eager triton kernel output."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.third_party.flash_linear_attention.ops.fused_norm_gate import (
|
||||
FusedRMSNormGated,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE = "xpu:0" if current_platform.is_xpu() else "cuda:0"
|
||||
|
||||
DTYPES = [torch.bfloat16]
|
||||
HIDDEN_SIZES = [128, 512]
|
||||
NUM_TOKENS = [64, 128]
|
||||
ACTIVATIONS = ["swish", "sigmoid"]
|
||||
ELEMENTWISE_AFFINE = [True, False]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("activation", ACTIVATIONS)
|
||||
@pytest.mark.parametrize("elementwise_affine", ELEMENTWISE_AFFINE)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_compiled_vs_eager(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
activation: str,
|
||||
elementwise_affine: bool,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
"""forward_native decomposition matches forward_cuda triton kernel."""
|
||||
torch._dynamo.reset()
|
||||
set_random_seed(seed)
|
||||
device = torch.device(DEVICE)
|
||||
|
||||
module = FusedRMSNormGated(
|
||||
hidden_size,
|
||||
elementwise_affine=elementwise_affine,
|
||||
eps=1e-5,
|
||||
activation=activation,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
# Model parameters use torch.empty because checkpoint loading overwrites
|
||||
# them. Initialize the standalone test module so allocator contents cannot
|
||||
# introduce NaNs and make this comparison flaky.
|
||||
if module.weight is not None:
|
||||
module.weight.uniform_(-1, 1)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
g = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
|
||||
# forward_cuda may modify x in-place, so clone inputs
|
||||
cuda_out = module.forward_cuda(x.clone(), g.clone())
|
||||
compiled_native = torch.compile(module.forward_native, fullgraph=True)
|
||||
native_out = compiled_native(x.clone(), g.clone())
|
||||
|
||||
torch.testing.assert_close(native_out, cuda_out, atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(1, 16, 32, 128),
|
||||
(2, 8, 16, 64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("activation", ACTIVATIONS)
|
||||
@pytest.mark.parametrize("elementwise_affine", ELEMENTWISE_AFFINE)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_compiled_vs_eager_multidim(
|
||||
default_vllm_config,
|
||||
shape: tuple,
|
||||
activation: str,
|
||||
elementwise_affine: bool,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
"""forward_native decomposition handles multi-dimensional inputs."""
|
||||
torch._dynamo.reset()
|
||||
set_random_seed(seed)
|
||||
device = torch.device(DEVICE)
|
||||
head_dim = shape[-1]
|
||||
|
||||
module = FusedRMSNormGated(
|
||||
head_dim,
|
||||
elementwise_affine=elementwise_affine,
|
||||
eps=1e-5,
|
||||
activation=activation,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
if module.weight is not None:
|
||||
module.weight.uniform_(-1, 1)
|
||||
x = torch.randn(*shape, dtype=dtype, device=device)
|
||||
g = torch.randn(*shape, dtype=dtype, device=device)
|
||||
|
||||
# forward_cuda may modify x in-place, so clone inputs
|
||||
cuda_out = module.forward_cuda(x.clone(), g.clone())
|
||||
compiled_native = torch.compile(module.forward_native, fullgraph=True)
|
||||
native_out = compiled_native(x.clone(), g.clone())
|
||||
|
||||
torch.testing.assert_close(native_out, cuda_out, atol=1e-3, rtol=1e-2)
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.utils import opcheck
|
||||
from tests.utils import large_gpu_test
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_group_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
QUANT_DTYPES = [current_platform.fp8_dtype(), torch.int8]
|
||||
VEC_HIDDEN_SIZES = [1024, 1025, 1027, 1029]
|
||||
NUM_TOKENS_HIDDEN_SIZES = [
|
||||
*[(1, i) for i in [64, *VEC_HIDDEN_SIZES, 2048, 5120]],
|
||||
*[(16, i) for i in [64, *VEC_HIDDEN_SIZES, 5120]],
|
||||
*[(128, i) for i in [64, *VEC_HIDDEN_SIZES]],
|
||||
*[(512, i) for i in [64, 5120]],
|
||||
]
|
||||
SCALE_UBS = [False]
|
||||
GROUP_SIZES = [64, 128]
|
||||
IS_SCALE_TRANSPOSED = [False, True]
|
||||
SEEDS = [0]
|
||||
CUDA_DEVICES = [i for i in range(1 if torch.accelerator.device_count() == 1 else 2)]
|
||||
|
||||
|
||||
def ref_silu_and_mul_per_block_quant(
|
||||
x: torch.Tensor,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference implementation: unfused SiLU+Mul then group quantization."""
|
||||
hidden = x.shape[-1] // 2
|
||||
gate, up = x.split(hidden, dim=-1)
|
||||
silu_out = F.silu(gate) * up
|
||||
|
||||
if quant_dtype == current_platform.fp8_dtype():
|
||||
return per_token_group_quant_fp8(
|
||||
silu_out, group_size=group_size, use_ue8m0=False
|
||||
)
|
||||
elif quant_dtype == torch.int8:
|
||||
return per_token_group_quant_int8(silu_out, group_size=group_size)
|
||||
else:
|
||||
raise ValueError(f"Unsupported quant_dtype: {quant_dtype}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens, hidden_size", NUM_TOKENS_HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("has_scale_ub", SCALE_UBS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_dtype", QUANT_DTYPES)
|
||||
@pytest.mark.parametrize("group_size", GROUP_SIZES)
|
||||
@pytest.mark.parametrize("is_scale_transposed", IS_SCALE_TRANSPOSED)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device_idx", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_silu_and_mul_per_block_quant(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
has_scale_ub: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_dtype: torch.dtype,
|
||||
group_size: int,
|
||||
is_scale_transposed: bool,
|
||||
seed: int,
|
||||
device_idx: str,
|
||||
) -> None:
|
||||
"""Test SiLU+Mul+Block Quantization kernel correctness."""
|
||||
torch.accelerator.set_device_index(device_idx)
|
||||
device = f"cuda:{device_idx}"
|
||||
torch.random.manual_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
|
||||
if hidden_size % group_size != 0:
|
||||
return
|
||||
|
||||
if has_scale_ub:
|
||||
pytest.skip("Scale upper bound not yet supported")
|
||||
|
||||
scale = 1 / hidden_size
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device) * scale
|
||||
|
||||
# Reference implementation
|
||||
ref_out, ref_scales = ref_silu_and_mul_per_block_quant(x, quant_dtype, group_size)
|
||||
|
||||
# Fused kernel implementation
|
||||
ops_out, ops_scales = ops.silu_and_mul_per_block_quant(
|
||||
x, group_size, quant_dtype, None, is_scale_transposed
|
||||
)
|
||||
|
||||
# Check for NaN/Inf
|
||||
assert not torch.isnan(ops_out.float()).any(), "Kernel output contains NaN"
|
||||
assert not torch.isinf(ops_out.float()).any(), "Kernel output contains Inf"
|
||||
assert not torch.isnan(ops_scales).any(), "Kernel scales contain NaN"
|
||||
assert not torch.isinf(ops_scales).any(), "Kernel scales contain Inf"
|
||||
|
||||
# Check dtypes
|
||||
assert ref_out.dtype == quant_dtype
|
||||
assert ops_out.dtype == quant_dtype
|
||||
|
||||
# Check scales match
|
||||
torch.testing.assert_close(ref_scales, ops_scales, rtol=1e-5, atol=1e-5)
|
||||
|
||||
# Check output correctness via dequantized values
|
||||
ref_scales_expanded = ref_scales.repeat_interleave(group_size, dim=1)
|
||||
ops_scales_expanded = ops_scales.repeat_interleave(group_size, dim=1)
|
||||
ref_deq = ref_out.to(dtype=torch.float32) * ref_scales_expanded
|
||||
ops_deq = ops_out.to(dtype=torch.float32) * ops_scales_expanded
|
||||
torch.testing.assert_close(ref_deq, ops_deq, atol=5e-2, rtol=5e-2)
|
||||
|
||||
# opcheck
|
||||
output = torch.empty(num_tokens, hidden_size, device=device, dtype=quant_dtype)
|
||||
num_groups = hidden_size // group_size
|
||||
if is_scale_transposed:
|
||||
scales = torch.empty(num_groups, num_tokens, device=device, dtype=torch.float32)
|
||||
else:
|
||||
scales = torch.empty(num_tokens, num_groups, device=device, dtype=torch.float32)
|
||||
opcheck(
|
||||
torch.ops._C.silu_and_mul_per_block_quant,
|
||||
(output, x, scales, group_size, None, is_scale_transposed),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16])
|
||||
@pytest.mark.parametrize("hidden_size", [4096])
|
||||
@pytest.mark.parametrize("num_tokens", [128])
|
||||
@pytest.mark.parametrize("group_size", [128])
|
||||
def test_silu_block_quant_shapes(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
hidden_size: int,
|
||||
num_tokens: int,
|
||||
group_size: int,
|
||||
):
|
||||
"""Test that output shapes are correct."""
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device="cuda")
|
||||
|
||||
# Row-major scales
|
||||
out, scales = ops.silu_and_mul_per_block_quant(
|
||||
x,
|
||||
group_size=group_size,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
is_scale_transposed=False,
|
||||
)
|
||||
assert out.shape == (num_tokens, hidden_size)
|
||||
assert scales.shape == (num_tokens, hidden_size // group_size)
|
||||
|
||||
# Column-major scales (logical shape same after .t() in _custom_ops)
|
||||
out, scales = ops.silu_and_mul_per_block_quant(
|
||||
x,
|
||||
group_size=group_size,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
is_scale_transposed=True,
|
||||
)
|
||||
assert out.shape == (num_tokens, hidden_size)
|
||||
assert scales.shape == (num_tokens, hidden_size // group_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16])
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 256])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 5120, 14336])
|
||||
def test_silu_block_quant_edge_cases(
|
||||
default_vllm_config, dtype: torch.dtype, batch_size: int, hidden_size: int
|
||||
):
|
||||
"""Test edge cases: single token, large batch, large hidden size."""
|
||||
torch.set_default_device("cuda")
|
||||
x = torch.randn(batch_size, hidden_size * 2, dtype=dtype, device="cuda")
|
||||
|
||||
out, scales = ops.silu_and_mul_per_block_quant(
|
||||
x,
|
||||
group_size=128,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
is_scale_transposed=False,
|
||||
)
|
||||
|
||||
assert out.shape == (batch_size, hidden_size)
|
||||
assert out.dtype == current_platform.fp8_dtype()
|
||||
assert scales.dtype == torch.float32
|
||||
assert not torch.isnan(out.float()).any()
|
||||
assert not torch.isnan(scales).any()
|
||||
assert not torch.isinf(scales).any()
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=24)
|
||||
@torch.inference_mode()
|
||||
def test_silu_block_quant_int32_offset_overflow(default_vllm_config):
|
||||
"""Token offsets beyond INT32_MAX must not wrap (issue #53390)."""
|
||||
hidden_size = 16384
|
||||
group_size = 128
|
||||
quant_dtype = current_platform.fp8_dtype()
|
||||
# Smallest batch whose last token's input offset (token_idx * 2 *
|
||||
# hidden_size) exceeds INT32_MAX.
|
||||
num_tokens = (2**31 - 1) // (2 * hidden_size) + 2
|
||||
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=torch.bfloat16, device="cuda")
|
||||
out, scales = ops.silu_and_mul_per_block_quant(
|
||||
x, group_size, quant_dtype, None, False
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_out, expected_scales = ops.silu_and_mul_per_block_quant(
|
||||
x[-1:].contiguous(), group_size, quant_dtype, None, False
|
||||
)
|
||||
assert torch.equal(scales[-1:], expected_scales)
|
||||
assert torch.equal(out[-1:], expected_out)
|
||||
@@ -1,260 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quant_utils import FP8_DTYPE
|
||||
from tests.kernels.utils import fp8_allclose, fp8_ulp_distance, opcheck
|
||||
from vllm import ir
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx90a
|
||||
|
||||
on_mi250 = on_gfx90a()
|
||||
else:
|
||||
on_mi250 = False
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16, torch.float]
|
||||
NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
|
||||
HIDDEN_SIZES = [8, 768, 769, 5120, 5125, 8192] # Arbitrary values for testing
|
||||
ADD_RESIDUAL = [False, True] if not on_mi250 else [True]
|
||||
SEEDS = [0]
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
|
||||
def _rms_norm_tolerance(dtype: torch.dtype) -> dict[str, float]:
|
||||
return ir.ops.rms_norm.get_tolerance(dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("strided_input", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
strided_input: bool,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
layer = RMSNorm(hidden_size).to(dtype=dtype)
|
||||
layer.weight.data.normal_(mean=1.0, std=0.1)
|
||||
scale = 1 / (2 * hidden_size)
|
||||
last_dim = 2 * hidden_size if strided_input else hidden_size
|
||||
x = torch.randn(num_tokens, last_dim, dtype=dtype)
|
||||
x = x[..., :hidden_size]
|
||||
assert x.is_contiguous() != strided_input
|
||||
x *= scale
|
||||
residual = x.new_empty_strided(x.size(), x.stride()) if add_residual else None
|
||||
if residual is not None:
|
||||
residual.normal_(std=scale)
|
||||
|
||||
# NOTE(woosuk): The reference implementation should be executed first
|
||||
# because the custom kernel is in-place.
|
||||
ref_out = layer.forward_native(x, residual)
|
||||
out = layer(x, residual)
|
||||
# NOTE(woosuk): LayerNorm operators (including RMS) typically have larger
|
||||
# numerical errors than other operators because they involve reductions.
|
||||
# Therefore, we use a larger tolerance.
|
||||
if add_residual:
|
||||
torch.testing.assert_close(out[0], ref_out[0], atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[1], ref_out[1], atol=1e-2, rtol=1e-2)
|
||||
else:
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
if residual is not None:
|
||||
opcheck(
|
||||
torch.ops._C.fused_add_rms_norm,
|
||||
(x, residual, layer.weight.data, layer.variance_epsilon),
|
||||
)
|
||||
else:
|
||||
opcheck(
|
||||
torch.ops._C.rms_norm, (out, x, layer.weight.data, layer.variance_epsilon)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_rms_norm_weightless(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
layer = RMSNorm(hidden_size, has_weight=False).to(dtype=dtype)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=dtype)
|
||||
residual = torch.randn_like(x) if add_residual else None
|
||||
|
||||
ref_out = layer.forward_native(x, residual)
|
||||
out = layer(x, residual)
|
||||
tol = _rms_norm_tolerance(dtype)
|
||||
if add_residual:
|
||||
torch.testing.assert_close(out[0], ref_out[0], **tol)
|
||||
torch.testing.assert_close(out[1], ref_out[1], **tol)
|
||||
else:
|
||||
torch.testing.assert_close(out, ref_out, **tol)
|
||||
|
||||
if residual is not None:
|
||||
opcheck(
|
||||
torch.ops._C.fused_add_rms_norm,
|
||||
(x, residual, None, layer.variance_epsilon),
|
||||
)
|
||||
else:
|
||||
opcheck(
|
||||
torch.ops._C.rms_norm,
|
||||
(out, x, None, layer.variance_epsilon),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("quant_scale", [0.01, 1.0, 10.0])
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("strided_input", [False, True])
|
||||
def test_fused_rms_norm_quant(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
add_residual: bool,
|
||||
dtype: torch.dtype,
|
||||
quant_scale: float,
|
||||
seed: int,
|
||||
device: str,
|
||||
strided_input: bool,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
|
||||
weight = torch.empty(hidden_size, dtype=dtype).normal_(mean=1.0, std=0.1)
|
||||
scale = 1 / (2 * hidden_size)
|
||||
last_dim = 2 * hidden_size if strided_input else hidden_size
|
||||
x_base = torch.randn(num_tokens, last_dim, dtype=dtype)
|
||||
x = x_base[..., :hidden_size]
|
||||
assert x.is_contiguous() != strided_input
|
||||
|
||||
x *= scale
|
||||
if add_residual:
|
||||
residual = torch.randn_like(x) * scale
|
||||
residual_fused = residual.clone()
|
||||
else:
|
||||
residual = residual_fused = None
|
||||
|
||||
out_norm = torch.empty_like(x)
|
||||
out_quant = torch.empty_like(x, dtype=FP8_DTYPE)
|
||||
out_quant_fused = torch.empty_like(out_quant)
|
||||
|
||||
quant_scale_t = torch.tensor(quant_scale, dtype=torch.float32)
|
||||
|
||||
if add_residual:
|
||||
torch.ops._C.fused_add_rms_norm_static_fp8_quant(
|
||||
out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6
|
||||
)
|
||||
|
||||
# Unfused kernel is in-place so it goes second
|
||||
# Also use a separate clone of x to avoid modifying the input
|
||||
x_unfused_base = x_base.clone()
|
||||
x_unfused = x_unfused_base[..., :hidden_size]
|
||||
assert x_unfused.is_contiguous() != strided_input
|
||||
torch.ops._C.fused_add_rms_norm(x_unfused, residual, weight, 1e-6)
|
||||
torch.ops._C.static_scaled_fp8_quant(
|
||||
out_quant, x_unfused.contiguous(), quant_scale_t
|
||||
)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(residual_fused, residual, atol=1e-2, rtol=1e-2)
|
||||
opcheck(
|
||||
torch.ops._C.fused_add_rms_norm_static_fp8_quant,
|
||||
(out_quant_fused, x, residual_fused, weight, quant_scale_t, 1e-6),
|
||||
)
|
||||
else:
|
||||
torch.ops._C.rms_norm_static_fp8_quant(
|
||||
out_quant_fused, x, weight, quant_scale_t, 1e-6
|
||||
)
|
||||
|
||||
torch.ops._C.rms_norm(out_norm, x, weight, 1e-6)
|
||||
torch.ops._C.static_scaled_fp8_quant(out_quant, out_norm, quant_scale_t)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.rms_norm_static_fp8_quant,
|
||||
(out_quant_fused, x, weight, quant_scale_t, 1e-6),
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if on_gfx950() and dtype == torch.float16 and not add_residual:
|
||||
# Fusion may round normalized FP16 across an E4M3 boundary on gfx950.
|
||||
assert fp8_allclose(out_quant_fused, out_quant, rtol=0.125, atol=2e-3)
|
||||
assert int(fp8_ulp_distance(out_quant_fused, out_quant).max()) <= 1
|
||||
else:
|
||||
# Fused and unfused FP8 paths can land on opposite sides of an E4M3
|
||||
# tie; tolerate a tiny number of isolated fp8 outliers on ROCm.
|
||||
ulp = fp8_ulp_distance(out_quant, out_quant_fused)
|
||||
max_outliers = ulp.numel() // 100_000 + 8
|
||||
num_outliers = int((ulp > 0).sum().item())
|
||||
assert num_outliers <= max_outliers, (
|
||||
f"FP8 quant mismatch: {num_outliers} fp8 outliers "
|
||||
f"(allowed {max_outliers})"
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
out_quant.to(dtype=torch.float32),
|
||||
out_quant_fused.to(dtype=torch.float32),
|
||||
atol=1e-3,
|
||||
rtol=1e-3,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gemma_rms_norm_mixed_input_weight_dtype(default_vllm_config) -> None:
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
|
||||
device = CUDA_DEVICES[0]
|
||||
torch.set_default_device(device)
|
||||
|
||||
num_tokens, hidden_size = 32, 1024
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
|
||||
layer = GemmaRMSNorm(hidden_size, eps=1e-6).to(device=device)
|
||||
layer.weight.data.normal_(mean=0.0, std=0.1)
|
||||
|
||||
# Gemma uses fp32 weight parameter while activations can be bf16.
|
||||
assert layer.weight.dtype == torch.float32
|
||||
out = layer(x)
|
||||
|
||||
x_fp32 = x.float()
|
||||
weight_fp32 = layer.weight.data.float() + 1.0
|
||||
variance = x_fp32.pow(2).mean(dim=-1, keepdim=True)
|
||||
ref = (x_fp32 * torch.rsqrt(variance + layer.variance_epsilon) * weight_fp32).to(
|
||||
x.dtype
|
||||
)
|
||||
|
||||
assert out.dtype == x.dtype
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
@@ -1,320 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MiniMax QK RMS-norm fused all-reduce kernels and fallbacks."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.multiprocessing import spawn
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.minimax_rms_norm import (
|
||||
MiniMaxText01RMSNormTP,
|
||||
rms_norm_tp,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@ensure_current_vllm_config()
|
||||
def _worker_forward_qk(
|
||||
local_rank,
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_q_full,
|
||||
hidden_k_full,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
):
|
||||
"""Per-rank worker: compare NCCL allreduce path vs Lamport fused kernel."""
|
||||
if not hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
|
||||
cleanup_dist_env_and_memory()
|
||||
return
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
|
||||
hq = hidden_q_full // world_size
|
||||
hk = hidden_k_full // world_size
|
||||
|
||||
q_norm = MiniMaxText01RMSNormTP(hidden_q_full, eps=eps).cuda()
|
||||
k_norm = MiniMaxText01RMSNormTP(hidden_k_full, eps=eps).cuda()
|
||||
|
||||
set_random_seed(seed)
|
||||
qw = torch.randn(hidden_q_full, dtype=dtype, device="cuda")
|
||||
kw = torch.randn(hidden_k_full, dtype=dtype, device="cuda")
|
||||
q_norm.weight = nn.Parameter(qw[local_rank * hq : (local_rank + 1) * hq])
|
||||
k_norm.weight = nn.Parameter(kw[local_rank * hk : (local_rank + 1) * hk])
|
||||
|
||||
torch.manual_seed(seed + 1000 + local_rank)
|
||||
qkv = torch.randn(num_tokens, hq + hk + hk, dtype=dtype, device="cuda")
|
||||
|
||||
# Reference: eager all-reduce path. ``forward_qk`` no longer all-reduces
|
||||
# the variance (it is the tp==1 / already-reduced building block), so the
|
||||
# multi-rank reference must use the eager path that performs the global
|
||||
# variance all-reduce, matching the fused kernel below.
|
||||
ref_q, ref_k = rms_norm_tp._minimax_qk_norm_tp_eager(
|
||||
qkv.clone(),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
hq,
|
||||
hk,
|
||||
world_size,
|
||||
eps,
|
||||
)
|
||||
|
||||
# Set up Lamport workspace.
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.model_executor.layers.minimax_rms_norm.lamport_workspace import (
|
||||
get_allreduce_workspace,
|
||||
)
|
||||
|
||||
workspace = get_allreduce_workspace(
|
||||
rank=local_rank,
|
||||
world_size=world_size,
|
||||
max_tokens=num_tokens,
|
||||
process_group=get_tp_group().cpu_group,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.minimax_allreduce_rms_qk,
|
||||
(
|
||||
qkv.clone(),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
workspace,
|
||||
hq,
|
||||
hk,
|
||||
local_rank,
|
||||
world_size,
|
||||
eps,
|
||||
),
|
||||
)
|
||||
fused_q, fused_k = torch.ops._C.minimax_allreduce_rms_qk(
|
||||
qkv.clone(),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
workspace,
|
||||
hq,
|
||||
hk,
|
||||
local_rank,
|
||||
world_size,
|
||||
eps,
|
||||
)
|
||||
_, _, fused_v = qkv.split([hq, hk, hk], dim=-1)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(
|
||||
fused_q,
|
||||
ref_q,
|
||||
atol=3e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
torch.testing.assert_close(fused_k, ref_k, atol=3e-2, rtol=3e-2)
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@ensure_current_vllm_config()
|
||||
def _worker_forward_qk_rocm_aiter(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
port: str,
|
||||
hidden_q_full: int,
|
||||
hidden_k_full: int,
|
||||
seed: int,
|
||||
eps: float,
|
||||
) -> None:
|
||||
"""Compare the ROCm AITER fusion with the eager TP implementation."""
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
|
||||
try:
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
assert aiter_ar is not None, "AITER custom all-reduce was not initialized"
|
||||
assert not aiter_ar.disabled, "AITER custom all-reduce is disabled"
|
||||
assert hasattr(aiter_ar.aiter_ca, "custom_fused_qknorm_ar")
|
||||
|
||||
hq = hidden_q_full // world_size
|
||||
hk = hidden_k_full // world_size
|
||||
test_cases = (
|
||||
(1, torch.bfloat16),
|
||||
(128, torch.bfloat16),
|
||||
(333, torch.bfloat16),
|
||||
(1, torch.float16),
|
||||
(128, torch.float16),
|
||||
(333, torch.float16),
|
||||
)
|
||||
|
||||
for num_tokens, dtype in test_cases:
|
||||
q_norm = MiniMaxText01RMSNormTP(hidden_q_full, eps=eps).cuda()
|
||||
k_norm = MiniMaxText01RMSNormTP(hidden_k_full, eps=eps).cuda()
|
||||
|
||||
set_random_seed(seed)
|
||||
q_weight = torch.randn(hidden_q_full, dtype=dtype, device="cuda")
|
||||
k_weight = torch.randn(hidden_k_full, dtype=dtype, device="cuda")
|
||||
q_norm.weight = nn.Parameter(
|
||||
q_weight[local_rank * hq : (local_rank + 1) * hq]
|
||||
)
|
||||
k_norm.weight = nn.Parameter(
|
||||
k_weight[local_rank * hk : (local_rank + 1) * hk]
|
||||
)
|
||||
|
||||
torch.manual_seed(seed + num_tokens + local_rank)
|
||||
qkv = torch.randn(num_tokens, hq + hk + hk, dtype=dtype, device="cuda")
|
||||
|
||||
ref_q, ref_k = rms_norm_tp._minimax_qk_norm_tp_eager(
|
||||
qkv.clone(),
|
||||
q_norm.weight,
|
||||
k_norm.weight,
|
||||
hq,
|
||||
hk,
|
||||
world_size,
|
||||
eps,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(rms_norm_tp, "_MINIMAX_FUSED_AR_RMS_QK", None),
|
||||
patch.object(
|
||||
rms_norm_tp,
|
||||
"_minimax_qk_norm_tp_fallback",
|
||||
side_effect=AssertionError(
|
||||
"ROCm AITER test unexpectedly used the fallback"
|
||||
),
|
||||
),
|
||||
):
|
||||
fused_q, fused_k, fused_v = MiniMaxText01RMSNormTP.forward_qkv(
|
||||
q_norm, k_norm, qkv.clone(), hq, hk
|
||||
)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(fused_q, ref_q, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(fused_k, ref_k, atol=3e-2, rtol=3e-2)
|
||||
assert torch.equal(fused_v, qkv[:, hq + hk :])
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.distributed(num_gpus=2)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm required",
|
||||
)
|
||||
def test_minimax_reduce_rms_qk_rocm_aiter(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The MiniMax TP QK path dispatches to AITER and matches eager RMSNorm."""
|
||||
assert is_aiter_found_and_supported(), "AITER requires ROCm CDNA3 or newer"
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
|
||||
|
||||
world_size = 2
|
||||
if current_platform.device_count() < world_size:
|
||||
pytest.skip("Need at least two GPUs")
|
||||
|
||||
spawn(
|
||||
_worker_forward_qk_rocm_aiter,
|
||||
args=(world_size, str(get_open_port()), 6144, 1024, 42, 1e-6),
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA required",
|
||||
)
|
||||
@pytest.mark.parametrize("world_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_dims",
|
||||
[(6144, 1024)],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.parametrize("seed", [42])
|
||||
def test_minimax_reduce_rms_qk(
|
||||
world_size,
|
||||
num_tokens,
|
||||
hidden_dims,
|
||||
dtype,
|
||||
eps,
|
||||
seed,
|
||||
):
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus < world_size:
|
||||
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
|
||||
hidden_q_full, hidden_k_full = hidden_dims
|
||||
port = str(get_open_port())
|
||||
spawn(
|
||||
_worker_forward_qk,
|
||||
args=(
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_q_full,
|
||||
hidden_k_full,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
),
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="CUDA and Triton required",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128, 333, 2049])
|
||||
@pytest.mark.parametrize("hidden_dims", [(3072, 512), (768, 256), (3000, 500)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("tp_world", [1, 4, 8])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.parametrize("seed", [42])
|
||||
def test_minimax_qk_norm_triton_fallback(
|
||||
monkeypatch, num_tokens, hidden_dims, dtype, tp_world, eps, seed
|
||||
):
|
||||
"""Single-GPU check: Triton fallback kernels vs the pure-torch reference.
|
||||
|
||||
The all-reduce is a TP communication barrier, so it is monkeypatched to
|
||||
identity here; both the Triton path and the reference see the same
|
||||
(patched) reduction. This validates the kernel math and the folded
|
||||
``/ tp_world`` scaling without needing multiple ranks -- ``hidden_dims``
|
||||
are the per-rank q/k segment widths.
|
||||
"""
|
||||
monkeypatch.setattr(rms_norm_tp, "_all_reduce_variance", lambda v: v)
|
||||
|
||||
q_size, kv_size = hidden_dims
|
||||
device = "cuda"
|
||||
torch.manual_seed(seed)
|
||||
qkv = torch.randn(num_tokens, q_size + 2 * kv_size, dtype=dtype, device=device)
|
||||
q_weight = torch.randn(q_size, dtype=dtype, device=device)
|
||||
k_weight = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
|
||||
q_triton, k_triton = rms_norm_tp._minimax_qk_norm_tp_fallback(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, 0, tp_world, eps
|
||||
)
|
||||
q_ref, k_ref = rms_norm_tp._minimax_qk_norm_tp_eager(
|
||||
qkv, q_weight, k_weight, q_size, kv_size, tp_world, eps
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_triton, q_ref, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(k_triton, k_ref, atol=3e-2, rtol=3e-2)
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import (
|
||||
apply_interleaved_rope,
|
||||
triton_mrope,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.config import get_config
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
def generate_test_data(
|
||||
num_tokens: int,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
max_position_embeddings: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
):
|
||||
"""Generate test data for given configuration."""
|
||||
set_random_seed(42)
|
||||
# Create 2D positions (3, num_tokens) for multimodal case
|
||||
positions = torch.randint(
|
||||
0, max_position_embeddings // 4, (3, num_tokens), device=device
|
||||
)
|
||||
|
||||
# Create query and key tensors
|
||||
query = torch.randn(num_tokens, num_q_heads * head_size, dtype=dtype, device=device)
|
||||
key = torch.randn(num_tokens, num_kv_heads * head_size, dtype=dtype, device=device)
|
||||
|
||||
return positions, query, key
|
||||
|
||||
|
||||
class MRoPETestInfo(NamedTuple):
|
||||
model_name: str
|
||||
is_neox_style: bool = True
|
||||
# https://github.com/pytorch/pytorch/blob/main/torch/testing/_comparison.py#L1317
|
||||
atol: float = 1e-2
|
||||
rtol: float = 1.6e-2
|
||||
marks: list[pytest.MarkDecorator] = []
|
||||
|
||||
|
||||
MODELS_TO_TEST = [
|
||||
MRoPETestInfo(
|
||||
model_name="zai-org/GLM-4.1V-9B-Thinking",
|
||||
is_neox_style=False,
|
||||
),
|
||||
MRoPETestInfo(model_name="Qwen/Qwen2-VL-7B-Instruct"),
|
||||
MRoPETestInfo(model_name="Qwen/Qwen2-VL-72B-Instruct"),
|
||||
MRoPETestInfo(model_name="Qwen/Qwen2.5-VL-72B-Instruct"),
|
||||
MRoPETestInfo(model_name="Qwen/Qwen3-VL-4B-Instruct"),
|
||||
MRoPETestInfo(model_name="Qwen/Qwen3-VL-30B-A3B-Instruct"),
|
||||
]
|
||||
|
||||
num_tokens_list = [11, 8192]
|
||||
|
||||
|
||||
def test_apply_interleaved_rope():
|
||||
mrope_section = [3, 1, 1]
|
||||
x = torch.tensor(
|
||||
[
|
||||
[[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]],
|
||||
[[10, 11, 12, 13, 14], [15, 16, 17, 18, 19]],
|
||||
[[20, 21, 22, 23, 24], [25, 26, 27, 28, 29]],
|
||||
]
|
||||
)
|
||||
|
||||
result = apply_interleaved_rope(x, mrope_section)
|
||||
|
||||
expected = torch.tensor([[0, 11, 22, 3, 4], [5, 16, 27, 8, 9]])
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only test."
|
||||
)
|
||||
def test_apply_interleaved_rope_torch_compile():
|
||||
mrope_section = [24, 20, 20]
|
||||
num_tokens = 8192
|
||||
rotary_dim = sum(mrope_section) * 2
|
||||
cache = torch.randn(
|
||||
3,
|
||||
num_tokens,
|
||||
rotary_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
x = cache[..., : rotary_dim // 2]
|
||||
|
||||
expected = apply_interleaved_rope(x, mrope_section)
|
||||
compiled_fn = torch.compile(
|
||||
apply_interleaved_rope,
|
||||
backend="inductor",
|
||||
fullgraph=True,
|
||||
)
|
||||
|
||||
result = compiled_fn(x, mrope_section)
|
||||
|
||||
torch.testing.assert_close(result, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model_info, model_name",
|
||||
[
|
||||
pytest.param(test_config, test_config.model_name, marks=test_config.marks)
|
||||
for test_config in MODELS_TO_TEST
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_tokens", num_tokens_list)
|
||||
def test_mrope(
|
||||
default_vllm_config,
|
||||
model_name: str,
|
||||
model_info: MRoPETestInfo,
|
||||
tp_size: int,
|
||||
dtype: torch.dtype,
|
||||
num_tokens: int,
|
||||
):
|
||||
atol = model_info.atol
|
||||
rtol = model_info.rtol
|
||||
|
||||
config = get_config(model_name, False).get_text_config()
|
||||
|
||||
# get the model config
|
||||
total_num_kv_heads = config.num_key_value_heads
|
||||
total_num_heads = config.num_attention_heads
|
||||
num_heads = total_num_heads // tp_size
|
||||
num_kv_heads = max(1, total_num_kv_heads // tp_size)
|
||||
head_dim = (
|
||||
config.head_dim
|
||||
if hasattr(config, "head_dim")
|
||||
else config.hidden_size // total_num_heads
|
||||
)
|
||||
is_neox_style = model_info.is_neox_style
|
||||
|
||||
max_position = config.max_position_embeddings
|
||||
|
||||
mrope_helper_class = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=is_neox_style,
|
||||
rope_parameters=config.rope_parameters,
|
||||
dtype=dtype,
|
||||
).to(device=device)
|
||||
|
||||
# create q k v input tensors
|
||||
# create rotary pos emb input tensors
|
||||
positions, query, key = generate_test_data(
|
||||
num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device
|
||||
)
|
||||
|
||||
query_native, key_native = mrope_helper_class.forward_native(
|
||||
positions,
|
||||
query.clone(),
|
||||
key.clone(),
|
||||
)
|
||||
|
||||
query_cuda, key_cuda = mrope_helper_class.forward_cuda(
|
||||
positions,
|
||||
query.clone(),
|
||||
key.clone(),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(query_native, query_cuda, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(key_native, key_cuda, atol=atol, rtol=rtol)
|
||||
|
||||
# Strided q/k views of a packed buffer (the MiniMax M3 ViT usage):
|
||||
# bitwise-identical to the contiguous run, neighboring columns untouched.
|
||||
pad = 16
|
||||
packed = torch.zeros(
|
||||
num_tokens,
|
||||
query.shape[1] + key.shape[1] + pad,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
packed[:, : query.shape[1]] = query
|
||||
packed[:, query.shape[1] : query.shape[1] + key.shape[1]] = key
|
||||
sentinel = packed[:, -pad:].clone()
|
||||
q_strided = packed[:, : query.shape[1]]
|
||||
k_strided = packed[:, query.shape[1] : query.shape[1] + key.shape[1]]
|
||||
mrope_helper_class.forward_cuda(positions, q_strided, k_strided)
|
||||
torch.testing.assert_close(q_strided, query_cuda, atol=0, rtol=0)
|
||||
torch.testing.assert_close(k_strided, key_cuda, atol=0, rtol=0)
|
||||
torch.testing.assert_close(packed[:, -pad:], sentinel, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
)
|
||||
def test_mrope_row_offset_int64():
|
||||
# MiniMax M3 ViT worst case: a max-size video item packs ~768k patches,
|
||||
# and num_tokens * packed-qkv row stride (3840) exceeds 2^31 at 559,241
|
||||
# rows. Row-base math must be int64 or those rows wrap to negative
|
||||
# offsets (illegal memory access at startup profiling).
|
||||
set_random_seed(0)
|
||||
n_qh, n_kh, hd, rd = 16, 16, 80, 78
|
||||
mrope_section = [13, 13, 13]
|
||||
row_stride = (n_qh + n_kh + n_kh) * hd # packed qkv row
|
||||
num_tokens = 2**31 // row_stride + 8
|
||||
free, _ = current_platform.mem_get_info()
|
||||
if free < 2 * num_tokens * row_stride * 2:
|
||||
pytest.skip("not enough free device memory")
|
||||
|
||||
packed = torch.empty(num_tokens, row_stride, dtype=torch.bfloat16, device=device)
|
||||
q_fill = torch.randn(n_qh * hd, dtype=torch.bfloat16, device=device)
|
||||
k_fill = torch.randn(n_kh * hd, dtype=torch.bfloat16, device=device)
|
||||
packed[:, : n_qh * hd] = q_fill
|
||||
packed[:, n_qh * hd : (n_qh + n_kh) * hd] = k_fill
|
||||
half_rd = rd // 2
|
||||
cos = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
|
||||
sin = torch.randn(3, num_tokens, half_rd, dtype=torch.float32, device=device)
|
||||
q = packed[:, : n_qh * hd]
|
||||
k = packed[:, n_qh * hd : (n_qh + n_kh) * hd]
|
||||
triton_mrope(q, k, cos, sin, mrope_section, hd, rd, False, True)
|
||||
|
||||
t_end = mrope_section[0]
|
||||
h_end = t_end + mrope_section[1]
|
||||
for r in (0, num_tokens - 8, num_tokens - 1):
|
||||
cos_row = torch.cat(
|
||||
[cos[0, r, :t_end], cos[1, r, t_end:h_end], cos[2, r, h_end:]]
|
||||
)
|
||||
sin_row = torch.cat(
|
||||
[sin[0, r, :t_end], sin[1, r, t_end:h_end], sin[2, r, h_end:]]
|
||||
)
|
||||
for fill, out in ((q_fill, q[r]), (k_fill, k[r])):
|
||||
row = fill.view(-1, hd).float()
|
||||
r1, r2 = row[:, :half_rd], row[:, half_rd:rd]
|
||||
exp = torch.cat(
|
||||
[r1 * cos_row - r2 * sin_row, r2 * cos_row + r1 * sin_row, row[:, rd:]],
|
||||
dim=1,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
out, exp.view(-1).to(torch.bfloat16), atol=1e-2, rtol=1.6e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skipping CUDA/ROCm only tests."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model_info, model_name",
|
||||
[
|
||||
pytest.param(test_config, test_config.model_name, marks=test_config.marks)
|
||||
for test_config in MODELS_TO_TEST
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_tokens", num_tokens_list)
|
||||
def test_mrope_torch_compile_tracing(
|
||||
default_vllm_config,
|
||||
model_name: str,
|
||||
model_info: MRoPETestInfo,
|
||||
tp_size: int,
|
||||
dtype: torch.dtype,
|
||||
num_tokens: int,
|
||||
):
|
||||
atol = model_info.atol
|
||||
rtol = model_info.rtol
|
||||
|
||||
config = get_config(model_name, False).get_text_config()
|
||||
|
||||
# get the model config
|
||||
total_num_kv_heads = config.num_key_value_heads
|
||||
total_num_heads = config.num_attention_heads
|
||||
num_heads = total_num_heads // tp_size
|
||||
num_kv_heads = max(1, total_num_kv_heads // tp_size)
|
||||
head_dim = (
|
||||
config.head_dim
|
||||
if hasattr(config, "head_dim")
|
||||
else config.hidden_size // total_num_heads
|
||||
)
|
||||
is_neox_style = model_info.is_neox_style
|
||||
max_position = config.max_position_embeddings
|
||||
|
||||
mrope_helper_class = get_rope(
|
||||
head_size=head_dim,
|
||||
max_position=max_position,
|
||||
is_neox_style=is_neox_style,
|
||||
rope_parameters=config.rope_parameters,
|
||||
dtype=dtype,
|
||||
).to(device=device)
|
||||
|
||||
# Generate test data
|
||||
positions, query, key = generate_test_data(
|
||||
num_tokens, num_heads, num_kv_heads, head_dim, max_position, dtype, device
|
||||
)
|
||||
|
||||
# Create a wrapper that makes the in-place function appear functional
|
||||
def functional_forward_cuda(pos, q, k):
|
||||
"""Wrapper that converts in-place operation to functional style.
|
||||
|
||||
CUDA Graph does not support in-place operations.
|
||||
This wrapper creates working copies of the
|
||||
input tensors and modifies them.
|
||||
"""
|
||||
q_work = q.clone() # Create working copies
|
||||
k_work = k.clone()
|
||||
# Your in-place function modifies q_work and k_work
|
||||
mrope_helper_class.forward_cuda(pos, q_work, k_work)
|
||||
return q_work, k_work # Return the modified tensors
|
||||
|
||||
# Get reference results
|
||||
query_native, key_native = mrope_helper_class.forward_native(
|
||||
positions,
|
||||
query.clone(),
|
||||
key.clone(),
|
||||
)
|
||||
|
||||
try:
|
||||
compiled_forward_cuda = torch.compile(
|
||||
functional_forward_cuda,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
# Run compiled version
|
||||
query_compiled_cuda, key_compiled_cuda = compiled_forward_cuda(
|
||||
positions,
|
||||
query,
|
||||
key,
|
||||
)
|
||||
|
||||
# Run original version for comparison
|
||||
query_cuda = query.clone()
|
||||
key_cuda = key.clone()
|
||||
mrope_helper_class.forward_cuda(positions, query_cuda, key_cuda)
|
||||
|
||||
# Verify results
|
||||
torch.testing.assert_close(
|
||||
query_compiled_cuda, query_cuda, atol=atol, rtol=rtol
|
||||
)
|
||||
torch.testing.assert_close(key_compiled_cuda, key_cuda, atol=atol, rtol=rtol)
|
||||
torch.testing.assert_close(
|
||||
query_compiled_cuda, query_native, atol=atol, rtol=rtol
|
||||
)
|
||||
torch.testing.assert_close(key_compiled_cuda, key_native, atol=atol, rtol=rtol)
|
||||
|
||||
print("✓ forward_cuda successfully traced with torch.compile inductor")
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"forward_cuda failed to trace with torch.compile inductor: {e}")
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from itertools import product
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
IS_NEOX_STYLE = [True, False]
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
HEAD_SIZES = [64, 80, 120, 256]
|
||||
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
|
||||
NUM_HEADS = [17] # Arbitrary values for testing
|
||||
BATCH_SIZES = [5] # Arbitrary values for testing
|
||||
SEQ_LENS = [11, 8192] # Arbitrary values for testing
|
||||
SEEDS = [0]
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
USE_KEY = [True, False]
|
||||
|
||||
|
||||
def _get_flat_tensor_shape(
|
||||
batch_size: int, seq_len: int, num_heads: int, head_size: int
|
||||
) -> tuple[int, ...]:
|
||||
return (batch_size, seq_len, num_heads * head_size)
|
||||
|
||||
|
||||
# For testing sliced tensors
|
||||
def _get_padded_tensor_shape(
|
||||
batch_size: int, seq_len: int, num_heads: int, head_size: int
|
||||
) -> tuple[int, ...]:
|
||||
return (batch_size, seq_len, num_heads, head_size + 64)
|
||||
|
||||
|
||||
def _get_batch_tensor_shape(
|
||||
batch_size: int, seq_len: int, num_heads: int, head_size: int
|
||||
) -> tuple[int, ...]:
|
||||
return (batch_size, seq_len, num_heads, head_size)
|
||||
|
||||
|
||||
TENSORS_SHAPES_FN = [
|
||||
_get_batch_tensor_shape,
|
||||
_get_flat_tensor_shape,
|
||||
_get_padded_tensor_shape,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
|
||||
@pytest.mark.parametrize("tensor_shape_fn", TENSORS_SHAPES_FN)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
@pytest.mark.parametrize("seq_len", SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("use_key", USE_KEY)
|
||||
@torch.inference_mode()
|
||||
def test_rotary_embedding(
|
||||
default_vllm_config,
|
||||
is_neox_style: bool,
|
||||
tensor_shape_fn: Callable[[int, int, int, int], tuple[int, ...]],
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
rotary_dim: int | None,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
device: str,
|
||||
use_key: bool,
|
||||
max_position: int = 8192,
|
||||
rope_theta: float = 10000,
|
||||
) -> None:
|
||||
if rotary_dim is None:
|
||||
rotary_dim = head_size
|
||||
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
if rotary_dim is None:
|
||||
rotary_dim = head_size
|
||||
rope_parameters = {
|
||||
"rope_type": "default",
|
||||
"rope_theta": rope_theta,
|
||||
"partial_rotary_factor": rotary_dim / head_size,
|
||||
}
|
||||
rope = get_rope(head_size, max_position, is_neox_style, rope_parameters)
|
||||
rope = rope.to(dtype=dtype, device=torch.get_default_device())
|
||||
|
||||
positions = torch.randint(0, max_position, (batch_size, seq_len))
|
||||
query_shape = tensor_shape_fn(batch_size, seq_len, num_heads, head_size)
|
||||
# slice tensor if required, noop otherwise
|
||||
query = torch.randn(query_shape, dtype=dtype)[..., :head_size]
|
||||
key = torch.randn_like(query)[..., :head_size] if use_key else None
|
||||
|
||||
# NOTE(woosuk): The reference implementation should be executed first
|
||||
# because the custom kernel is in-place.
|
||||
ref_query, ref_key = rope.forward_native(positions, query, key)
|
||||
out_query, out_key = rope.forward(positions, query, key)
|
||||
# Compare the results.
|
||||
torch.testing.assert_close(
|
||||
out_query,
|
||||
ref_query,
|
||||
atol=get_default_atol(out_query),
|
||||
rtol=get_default_rtol(out_query),
|
||||
)
|
||||
if use_key:
|
||||
torch.testing.assert_close(
|
||||
out_key,
|
||||
ref_key,
|
||||
atol=get_default_atol(out_key),
|
||||
rtol=get_default_rtol(out_key),
|
||||
)
|
||||
else:
|
||||
assert ref_key is None and out_key is None, "expected returned key to be None"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_rope_module_cache(default_vllm_config):
|
||||
MAX_POSITIONS = [123, 1234]
|
||||
ROPE_THETAS = [10000, 1000000]
|
||||
ROPE_PARAMETERS = (
|
||||
{"rope_type": "default"},
|
||||
{"rope_type": "linear", "factor": (1,)},
|
||||
{"rope_type": "dynamic", "factor": 1},
|
||||
)
|
||||
settings = (
|
||||
HEAD_SIZES,
|
||||
ROTARY_DIMS,
|
||||
MAX_POSITIONS,
|
||||
ROPE_THETAS,
|
||||
IS_NEOX_STYLE,
|
||||
ROPE_PARAMETERS,
|
||||
DTYPES,
|
||||
)
|
||||
rope_setting_id_map: dict[str, int] = {}
|
||||
for setting in product(*settings):
|
||||
(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
rope_theta,
|
||||
is_neox_style,
|
||||
rope_parameters,
|
||||
dtype,
|
||||
) = setting
|
||||
if rotary_dim is None:
|
||||
rotary_dim = head_size
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
|
||||
rope = get_rope(
|
||||
head_size,
|
||||
max_position,
|
||||
is_neox_style,
|
||||
rope_parameters,
|
||||
dtype,
|
||||
)
|
||||
# different settings cannot share the same rope module
|
||||
assert id(rope) not in rope_setting_id_map.values()
|
||||
assert all(x.dtype == dtype for x in rope.buffers())
|
||||
assert all(x.dtype == dtype for x in rope.parameters())
|
||||
rope_setting_id_map[str(setting)] = id(rope)
|
||||
|
||||
for setting in product(*settings):
|
||||
(
|
||||
head_size,
|
||||
rotary_dim,
|
||||
max_position,
|
||||
rope_theta,
|
||||
is_neox_style,
|
||||
rope_parameters,
|
||||
dtype,
|
||||
) = setting
|
||||
if rotary_dim is None:
|
||||
rotary_dim = head_size
|
||||
rope_parameters["rope_theta"] = rope_theta
|
||||
rope_parameters["partial_rotary_factor"] = rotary_dim / head_size
|
||||
rope = get_rope(
|
||||
head_size,
|
||||
max_position,
|
||||
is_neox_style,
|
||||
rope_parameters,
|
||||
dtype,
|
||||
)
|
||||
# check if cache take effect
|
||||
assert id(rope) == rope_setting_id_map[str(setting)]
|
||||
@@ -1,684 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm AITER helper-op tests for CI stability.
|
||||
|
||||
These tests directly validate AITER kernel correctness against PyTorch/reference
|
||||
implementations. They provide clear failure signals when AITER library updates
|
||||
introduce regressions, independent of vLLM's fusion/compilation passes.
|
||||
|
||||
Tested ops:
|
||||
- ``rms_norm`` and ``rms_norm2d_with_add`` vs PyTorch reference
|
||||
- ``triton_rotary_embedding`` vs manual NeoX RoPE reference (xfail - known issue)
|
||||
- ``act_mul_and_fp8_group_quant`` (SiGLU + FP8 group quant)
|
||||
- fused RMSNorm + quantization ops vs sequential composition
|
||||
|
||||
Related coverage:
|
||||
- per-token/per-tensor quant roundtrips: ``tests/rocm/aiter/test_quant_op_schema.py``
|
||||
- RMSNorm determinism: ``tests/rocm/aiter/test_quant_op_schema.py``
|
||||
- group_fp8_quant: ``tests/kernels/quantization/test_rocm_aiter_grouped_quant.py``
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._aiter_ops # noqa: F401 - ensure ops are registered
|
||||
from tests.kernels.utils import bf16_ulp_distance, fp8_ulp_distance
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda_device():
|
||||
"""Set default device to CUDA and seed RNG for all tests."""
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
|
||||
|
||||
def require_aiter():
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("aiter is not found or not supported on this hardware")
|
||||
|
||||
|
||||
def require_fp8():
|
||||
if not current_platform.supports_fp8():
|
||||
pytest.skip("FP8 is not supported on this hardware")
|
||||
|
||||
|
||||
def _format_observed_rate(count: int, total: int) -> str:
|
||||
return f"{count / total:.4%} ({count}/{total})"
|
||||
|
||||
|
||||
def _format_allowed_rate(rate: float, total: int) -> str:
|
||||
allowed_count = int(rate * total)
|
||||
return f"{rate:.4%} (<= {allowed_count}/{total})"
|
||||
|
||||
|
||||
def _quantile(values: torch.Tensor, q: float) -> float:
|
||||
if values.numel() == 0:
|
||||
return 0.0
|
||||
return torch.quantile(values, q).item()
|
||||
|
||||
|
||||
def _dequantize_grouped(
|
||||
quantized: torch.Tensor, scales: torch.Tensor, group_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Expand per-group scales and dequantize to float32."""
|
||||
M, N = quantized.shape
|
||||
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :N]
|
||||
return quantized.float() * scales_expanded
|
||||
|
||||
|
||||
def _rms_norm_reference(
|
||||
x: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
"""RMSNorm reference implementation in float32."""
|
||||
rms = x.float().pow(2).mean(-1, keepdim=True).add(eps).sqrt()
|
||||
return x.float() / rms * weight.float()
|
||||
|
||||
|
||||
def _assert_quant_shapes(
|
||||
x_quant: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
M: int,
|
||||
N: int,
|
||||
fp8_dtype: torch.dtype,
|
||||
num_groups: int | None = None,
|
||||
) -> None:
|
||||
"""Verify quantized output shapes and dtypes."""
|
||||
assert x_quant.shape == (M, N)
|
||||
assert x_quant.dtype == fp8_dtype
|
||||
if num_groups is not None:
|
||||
assert scale.shape == (M, num_groups)
|
||||
else:
|
||||
assert scale.shape == (M, 1)
|
||||
|
||||
|
||||
def _assert_close_budget(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
atol: float,
|
||||
rtol: float = 0.0,
|
||||
pass_rate: float = 0.99999,
|
||||
max_violation_factor: float = 3.0,
|
||||
) -> None:
|
||||
actual_f = actual.detach().float().flatten()
|
||||
expected_f = expected.detach().float().flatten()
|
||||
abs_diff = (actual_f - expected_f).abs()
|
||||
allowed = atol + rtol * expected_f.abs()
|
||||
|
||||
total = abs_diff.numel()
|
||||
passed = int((abs_diff <= allowed).sum().item())
|
||||
failed = total - passed
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
max_abs = abs_diff.max().item()
|
||||
mean_abs = abs_diff.mean().item()
|
||||
p99_abs = _quantile(abs_diff, 0.99)
|
||||
p999_abs = _quantile(abs_diff, 0.999)
|
||||
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
|
||||
max_atol = max_violation_factor * atol
|
||||
above_max_count = int((abs_diff > max_atol).sum().item())
|
||||
|
||||
msg = (
|
||||
"[rocm_aiter_ops] "
|
||||
f"{label}: "
|
||||
f"pass={passed / total:.4%} ({passed}/{total}) "
|
||||
f"fail={_format_observed_rate(failed, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"atol={atol:g} "
|
||||
f"rtol={rtol:g} "
|
||||
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
|
||||
f"max_abs={max_abs:.6g} "
|
||||
f"mean_abs={mean_abs:.6g} "
|
||||
f"p99_abs={p99_abs:.6g} "
|
||||
f"p999_abs={p999_abs:.6g} "
|
||||
f"worst_ratio={worst_ratio:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if failed > 0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
assert passed / total >= pass_rate, msg
|
||||
assert max_abs <= max_atol, msg
|
||||
assert mean_abs <= atol * 0.25, msg
|
||||
|
||||
|
||||
def _assert_rel_error_quality(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
mean_limit: float,
|
||||
preferred_rel: float,
|
||||
max_rel: float,
|
||||
max_fail_rate: float,
|
||||
) -> None:
|
||||
rel = (
|
||||
(actual.float() - expected.float()).abs()
|
||||
/ expected.float().abs().clamp_min(1e-5)
|
||||
).flatten()
|
||||
total = rel.numel()
|
||||
within_preferred_count = int((rel <= preferred_rel).sum().item())
|
||||
fail_count = total - within_preferred_count
|
||||
above_max_count = int((rel > max_rel).sum().item())
|
||||
mean_rel = rel.mean().item()
|
||||
max_rel_err = rel.max().item()
|
||||
p99 = _quantile(rel, 0.99)
|
||||
p999 = _quantile(rel, 0.999)
|
||||
|
||||
msg = (
|
||||
"[rocm_aiter_ops] "
|
||||
f"{label}: "
|
||||
f"rel<={preferred_rel:g} pass={within_preferred_count / total:.4%} "
|
||||
f"({within_preferred_count}/{total}) "
|
||||
f"fail={_format_observed_rate(fail_count, total)} "
|
||||
f"mean_limit={mean_limit:.4%} "
|
||||
f"rel>{max_rel:g}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
|
||||
f"mean_rel={mean_rel:.6g} "
|
||||
f"max_rel={max_rel_err:.6g} "
|
||||
f"p99={p99:.6g} "
|
||||
f"p999={p999:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if fail_count > 0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
assert mean_rel < mean_limit, msg
|
||||
assert above_max_count / total <= max_fail_rate, msg
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_aiter_rms_norm_vs_torch(dtype):
|
||||
"""rocm_aiter_rms_norm matches PyTorch manual RMSNorm for float16 and bfloat16."""
|
||||
require_aiter()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 8, 128
|
||||
x = torch.randn(M, N, dtype=dtype)
|
||||
weight = torch.ones(N, dtype=dtype)
|
||||
eps = 1e-5
|
||||
|
||||
# Reference: float32 RMSNorm for precision
|
||||
ref = _rms_norm_reference(x, weight, eps).to(dtype)
|
||||
|
||||
out = rocm_aiter_ops.rms_norm(x, weight, eps)
|
||||
|
||||
# Allow max 1 ULP difference due to different accumulation order.
|
||||
ulp = bf16_ulp_distance(out, ref)
|
||||
max_ulp = int(ulp.max().item())
|
||||
assert max_ulp <= 1, f"rms_norm dtype={dtype}: max ULP distance {max_ulp} > 1"
|
||||
|
||||
|
||||
# -- Numerical accuracy tests for AITER custom ops -------------------------
|
||||
|
||||
|
||||
def test_rocm_aiter_rmsnorm_with_add_vs_torch():
|
||||
"""rocm_aiter_rmsnorm2d_fwd_with_add matches manual residual+RMSNorm reference."""
|
||||
require_aiter()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 16, 256
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
residual = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.ones(N, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
|
||||
# Reference: add residual, then RMSNorm
|
||||
h = x.float() + residual.float()
|
||||
rms = h.pow(2).mean(-1, keepdim=True).add(eps).sqrt()
|
||||
ref_normed = (h / rms * weight.float()).to(torch.bfloat16)
|
||||
ref_residual = h.to(torch.bfloat16)
|
||||
|
||||
out, res_out = rocm_aiter_ops.rms_norm2d_with_add(x, residual, weight, eps)
|
||||
|
||||
# Allow max 1 ULP difference due to different accumulation order.
|
||||
ulp_normed = bf16_ulp_distance(out, ref_normed)
|
||||
max_ulp_normed = int(ulp_normed.max().item())
|
||||
assert max_ulp_normed <= 1, (
|
||||
f"rmsnorm_with_add normed: max ULP distance {max_ulp_normed} > 1"
|
||||
)
|
||||
|
||||
ulp_residual = bf16_ulp_distance(res_out, ref_residual)
|
||||
max_ulp_residual = int(ulp_residual.max().item())
|
||||
assert max_ulp_residual <= 1, (
|
||||
f"rmsnorm_with_add residual: max ULP distance {max_ulp_residual} > 1"
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_triton_rotary_embedding_vs_torch():
|
||||
"""rocm_aiter_triton_rotary_embedding matches manual NeoX-style RoPE reference.
|
||||
|
||||
The AITER kernel calls rope_cached_thd_positions_offsets_2c_fwd_inplace with
|
||||
reuse_freqs_front_part=True. This means it reads only the first head_size//2
|
||||
entries of cos and sin from the cache and applies them to both halves of the
|
||||
head (pairwise NeoX rotation). The cos_sin_cache must be built with the second
|
||||
half mirroring the first so the reference and kernel agree.
|
||||
"""
|
||||
require_aiter()
|
||||
|
||||
num_tokens = 8
|
||||
num_heads = 4
|
||||
head_size = 64
|
||||
half_dim = head_size // 2
|
||||
max_pos = 32
|
||||
|
||||
positions = torch.randint(0, max_pos, (num_tokens,), dtype=torch.long)
|
||||
query = torch.randn(num_tokens, num_heads * head_size, dtype=torch.bfloat16)
|
||||
key = torch.randn(num_tokens, num_heads * head_size, dtype=torch.bfloat16)
|
||||
|
||||
# Build cos/sin cache with second half mirroring first half.
|
||||
# The kernel uses reuse_freqs_front_part=True: it reads only cos/sin[:, :half_dim]
|
||||
# and applies those same frequencies to both the first and second
|
||||
# halves of the head.
|
||||
cos_half = torch.randn(max_pos, half_dim, dtype=torch.bfloat16)
|
||||
sin_half = torch.randn(max_pos, half_dim, dtype=torch.bfloat16)
|
||||
cos_cache = torch.cat([cos_half, cos_half], dim=-1) # [max_pos, head_size]
|
||||
sin_cache = torch.cat([sin_half, sin_half], dim=-1) # [max_pos, head_size]
|
||||
cos_sin_cache = torch.cat([cos_cache, sin_cache], dim=-1) # [max_pos, 2*head_size]
|
||||
|
||||
# Reference: NeoX-style pairwise rotation with front-half frequencies only.
|
||||
# rotate_style=0 (NeoX): [x1*c - x2*s, x2*c + x1*s]
|
||||
cos_pos = cos_half[positions] # [num_tokens, half_dim]
|
||||
sin_pos = sin_half[positions] # [num_tokens, half_dim]
|
||||
|
||||
def apply_rope_ref(t: torch.Tensor) -> torch.Tensor:
|
||||
# AITER triton keeps native dtype (no fp32 upcast), so reference must match.
|
||||
t_r = t.view(num_tokens, num_heads, head_size)
|
||||
c = cos_pos.unsqueeze(1) # [num_tokens, 1, half_dim]
|
||||
s = sin_pos.unsqueeze(1)
|
||||
x1, x2 = t_r[..., :half_dim], t_r[..., half_dim:]
|
||||
rotated = torch.cat([x1 * c - x2 * s, x2 * c + x1 * s], dim=-1)
|
||||
return rotated.view(num_tokens, num_heads * head_size)
|
||||
|
||||
ref_q = apply_rope_ref(query)
|
||||
ref_k = apply_rope_ref(key)
|
||||
|
||||
# AITER in-place RoPE (modifies query/key in-place)
|
||||
q_aiter = query.clone()
|
||||
k_aiter = key.clone()
|
||||
torch.ops.vllm.rocm_aiter_triton_rotary_embedding(
|
||||
positions,
|
||||
q_aiter,
|
||||
k_aiter,
|
||||
head_size,
|
||||
cos_sin_cache,
|
||||
True, # is_neox style -> rotate_style=0
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_aiter, ref_q)
|
||||
torch.testing.assert_close(k_aiter, ref_k)
|
||||
|
||||
|
||||
def test_rocm_aiter_act_mul_fp8_group_quant_roundtrip():
|
||||
"""act_mul_and_fp8_group_quant: dequantized output matches SiLU gate reference."""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
|
||||
M, N = 32, 512 # N even: N//2 gate, N//2 up
|
||||
group_size = 128
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
|
||||
x_quant, scale = torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant(
|
||||
x, group_size
|
||||
)
|
||||
|
||||
N_half = N // 2
|
||||
# Reference: SiLU(gate) * up
|
||||
gate = x.float()[:, :N_half]
|
||||
up = x.float()[:, N_half:]
|
||||
ref = torch.sigmoid(gate) * gate * up # SiGLU
|
||||
|
||||
x_dequant = _dequantize_grouped(x_quant, scale, group_size)
|
||||
|
||||
_assert_rel_error_quality(
|
||||
x_dequant,
|
||||
ref,
|
||||
label="act_mul_fp8_group_quant",
|
||||
mean_limit=0.1,
|
||||
preferred_rel=0.1,
|
||||
max_rel=1.0,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
# -- Fused RMSNorm + quantization accuracy tests ---------------------------
|
||||
|
||||
|
||||
def test_rocm_aiter_rmsnorm_fused_dynamic_quant_vs_sequential():
|
||||
"""Fused RMSNorm+per-token-FP8-quant matches sequential rms_norm->per_token_quant.
|
||||
|
||||
Tests that the fused kernel produces the same result as the two-step
|
||||
sequential composition: rms_norm(x) followed by per_token_quant.
|
||||
The fused path is used in production for inference throughput.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 32, 512
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.ones(N, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
# Sequential reference
|
||||
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
|
||||
ref_q, ref_scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
|
||||
ref_dequant = ref_q.float() * ref_scale.float()
|
||||
|
||||
# Fused op
|
||||
fused_q, fused_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant(
|
||||
x, weight, eps, fp8_dtype
|
||||
)
|
||||
_assert_quant_shapes(fused_q, fused_scale, M, N, fp8_dtype)
|
||||
fused_dequant = fused_q.float() * fused_scale.float()
|
||||
|
||||
# Fused vs sequential: both should recover RMSNorm output within FP8 error
|
||||
_assert_rel_error_quality(
|
||||
fused_dequant,
|
||||
ref_dequant,
|
||||
label="rmsnorm_fused_dynamic_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_rmsnorm_fused_add_dynamic_quant_vs_reference():
|
||||
"""Fused (residual-add + RMSNorm + per-token-FP8-quant)
|
||||
matches sequential reference.
|
||||
|
||||
Production path: input + residual -> RMSNorm -> FP8 quantize, returning
|
||||
both the quantized norm output and the residual sum for the next layer.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 16, 256
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
residual = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.ones(N, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
# Sequential reference: add residual -> rms_norm -> per_token_quant
|
||||
h = (x.float() + residual.float()).to(torch.bfloat16)
|
||||
normed = rocm_aiter_ops.rms_norm(h, weight, eps)
|
||||
ref_q, ref_scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
|
||||
ref_residual_out = h
|
||||
|
||||
# Fused op: returns (x_quant, residual_out, scale)
|
||||
fused_q, fused_res_out, fused_scale = (
|
||||
torch.ops.vllm.rocm_aiter_rmsnorm_fused_add_dynamic_quant(
|
||||
x, residual, weight, eps, fp8_dtype
|
||||
)
|
||||
)
|
||||
_assert_quant_shapes(fused_q, fused_scale, M, N, fp8_dtype)
|
||||
assert fused_res_out.shape == (M, N)
|
||||
|
||||
torch.testing.assert_close(fused_res_out, ref_residual_out, atol=1e-2, rtol=1.6e-2)
|
||||
# Dequantized output matches sequential path
|
||||
fused_dequant = fused_q.float() * fused_scale.float()
|
||||
ref_dequant = ref_q.float() * ref_scale.float()
|
||||
_assert_rel_error_quality(
|
||||
fused_dequant,
|
||||
ref_dequant,
|
||||
label="rmsnorm_fused_add_dynamic_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_rmsnorm_fp8_group_quant_vs_sequential():
|
||||
"""Fused RMSNorm+FP8-group-quant matches sequential rms_norm->group_fp8_quant.
|
||||
|
||||
Tests both output shapes and dequantized accuracy against the two-step
|
||||
sequential composition.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 32, 512
|
||||
group_size = 128
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.ones(N, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
expected_groups = (N + group_size - 1) // group_size
|
||||
|
||||
# Fused op: (x_quant, scales)
|
||||
fused_q, fused_scales = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
|
||||
x, weight, eps, group_size
|
||||
)
|
||||
_assert_quant_shapes(fused_q, fused_scales, M, N, fp8_dtype, expected_groups)
|
||||
|
||||
# Dequantize and compare to reference: rms_norm -> group quant -> dequant
|
||||
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
|
||||
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed, group_size)
|
||||
ref_dequant = _dequantize_grouped(ref_q, ref_scales, group_size)
|
||||
fused_dequant = _dequantize_grouped(fused_q, fused_scales, group_size)
|
||||
|
||||
_assert_rel_error_quality(
|
||||
fused_dequant,
|
||||
ref_dequant,
|
||||
label="rmsnorm_fp8_group_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_rmsnorm_with_add_fp8_group_quant_residual_accuracy():
|
||||
"""Fused rmsnorm_with_add_fp8_group_quant residual output matches x + residual."""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 16, 256
|
||||
group_size = 128
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
residual = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.ones(N, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
|
||||
fused_q, fused_res, fused_scales = (
|
||||
torch.ops.vllm.rocm_aiter_rmsnorm_with_add_fp8_group_quant(
|
||||
x, residual, weight, eps, group_size
|
||||
)
|
||||
)
|
||||
|
||||
# Residual output must equal x + residual
|
||||
ref_residual = (x.float() + residual.float()).to(torch.bfloat16)
|
||||
_assert_close_budget(
|
||||
fused_res.float(),
|
||||
ref_residual.float(),
|
||||
label="rmsnorm_with_add_fp8_group_quant residual",
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
|
||||
# Dequantized quant output must match rms_norm(x + residual)
|
||||
h = ref_residual
|
||||
ref_normed = _rms_norm_reference(h, weight, eps).to(torch.bfloat16)
|
||||
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(ref_normed, group_size)
|
||||
ref_dequant = _dequantize_grouped(ref_q, ref_scales, group_size)
|
||||
fused_dequant = _dequantize_grouped(fused_q, fused_scales, group_size)
|
||||
_assert_rel_error_quality(
|
||||
fused_dequant,
|
||||
ref_dequant,
|
||||
label="rmsnorm_with_add_fp8_group_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 32])
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
|
||||
"""Fused q/kv RMSNorm + q group-quant matches rms_norm -> group_fp8_quant.
|
||||
|
||||
M=1 covers the single-token decode case. M=0 is guarded by the caller
|
||||
(the ROCm DeepSeek-V4 attention checks num_tokens > 0) and the op has
|
||||
no zero-row guard itself, so it is intentionally not tested here.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
nq, nk, group_size = 1536, 576, 128
|
||||
q = torch.randn(m, nq, dtype=torch.bfloat16)
|
||||
kv = torch.randn(m, nk, dtype=torch.bfloat16)
|
||||
q_weight = torch.ones(nq, dtype=torch.bfloat16)
|
||||
kv_weight = torch.ones(nk, dtype=torch.bfloat16)
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
fused_q, fused_scales, fused_kv = rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
|
||||
)
|
||||
_assert_quant_shapes(fused_q, fused_scales, m, nq, fp8_dtype, nq // group_size)
|
||||
assert fused_kv.shape == (m, nk)
|
||||
assert fused_kv.dtype == kv.dtype
|
||||
|
||||
# q: dequantized comparison against the sequential composition.
|
||||
normed_q = rocm_aiter_ops.rms_norm(q, q_weight, eps)
|
||||
ref_q, ref_scales = rocm_aiter_ops.group_fp8_quant(normed_q, group_size)
|
||||
_assert_rel_error_quality(
|
||||
_dequantize_grouped(fused_q, fused_scales, group_size),
|
||||
_dequantize_grouped(ref_q, ref_scales, group_size),
|
||||
label="mla_dual_rmsnorm_group_quant",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
# kv: both sides are aiter RMSNorm kernels; allow one bf16 ULP.
|
||||
ref_kv = rocm_aiter_ops.rms_norm(kv, kv_weight, eps)
|
||||
max_ulp = int(bf16_ulp_distance(fused_kv, ref_kv).max().item())
|
||||
assert max_ulp <= 1, f"fused kv: max ULP {max_ulp} > 1"
|
||||
|
||||
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
||||
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
||||
contiguous inputs."""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
m, nq, nk, group_size = 16, 1536, 576, 128
|
||||
qr_kv = torch.randn(m, nq + nk, dtype=torch.bfloat16)
|
||||
q_view, kv_view = qr_kv.split([nq, nk], dim=-1)
|
||||
assert not q_view.is_contiguous()
|
||||
q_weight, kv_weight = torch.ones(nq + nk, dtype=torch.bfloat16).split([nq, nk])
|
||||
eps = 1e-5
|
||||
|
||||
def run(q, kv):
|
||||
return rocm_aiter_ops.fused_qk_rmsnorm_group_quant(
|
||||
q, q_weight, eps, kv, kv_weight, eps, group_size=group_size
|
||||
)
|
||||
|
||||
out_strided = run(q_view, kv_view)
|
||||
out_contig = run(q_view.contiguous(), kv_view.contiguous())
|
||||
|
||||
q_ulp = int(fp8_ulp_distance(out_strided[0], out_contig[0]).max().item())
|
||||
kv_ulp = int(bf16_ulp_distance(out_strided[2], out_contig[2]).max().item())
|
||||
assert q_ulp <= 1, f"strided q: max fp8 ULP {q_ulp} > 1"
|
||||
assert kv_ulp <= 1, f"strided kv: max bf16 ULP {kv_ulp} > 1"
|
||||
torch.testing.assert_close(out_strided[1], out_contig[1])
|
||||
|
||||
|
||||
# -- End-to-end inference chain test ---------------------------------------
|
||||
|
||||
|
||||
def test_rocm_aiter_rms_norm_then_per_token_quant_e2e():
|
||||
"""End-to-end: BF16 RMSNorm -> per-token FP8 quantization -> dequantize.
|
||||
|
||||
Simulates the inference path through a transformer layer norm before a
|
||||
linear projection: verifies the full chain produces accurate output
|
||||
compared to a float32 reference.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Llama-style hidden dim
|
||||
M, N = 32, 4096
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, dtype=torch.bfloat16) # learned scale
|
||||
eps = 1e-5
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
|
||||
# Float32 reference for the full chain
|
||||
ref_normed_f32 = _rms_norm_reference(x, weight, eps)
|
||||
|
||||
# AITER chain: RMSNorm -> per-token FP8 quant -> dequant
|
||||
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
|
||||
x_q, scale = rocm_aiter_ops.per_token_quant(normed, fp8_dtype)
|
||||
x_dequant = x_q.float() * scale.float() # scale: [M, 1]
|
||||
|
||||
_assert_quant_shapes(x_q, scale, M, N, fp8_dtype)
|
||||
# Dequantized result should match the float32 reference within FP8 quant error
|
||||
_assert_rel_error_quality(
|
||||
x_dequant,
|
||||
ref_normed_f32,
|
||||
label="rms_norm_then_per_token_quant_e2e",
|
||||
mean_limit=0.05,
|
||||
preferred_rel=0.05,
|
||||
max_rel=0.5,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_aiter_rms_norm_then_group_fp8_quant_e2e(dtype):
|
||||
"""End-to-end: RMSNorm (fp16/bf16) -> FP8 group quantization -> dequantize.
|
||||
|
||||
Covers both float16 and bfloat16 inputs to verify dtype-agnostic
|
||||
behavior of the RMSNorm+group-quant pipeline.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
M, N = 16, 512
|
||||
group_size = 128
|
||||
x = torch.randn(M, N, dtype=dtype)
|
||||
weight = torch.randn(N, dtype=dtype)
|
||||
eps = 1e-5
|
||||
|
||||
# Float32 reference
|
||||
ref_normed_f32 = _rms_norm_reference(x, weight, eps)
|
||||
|
||||
# AITER chain: RMSNorm -> group FP8 quant -> dequant
|
||||
normed = rocm_aiter_ops.rms_norm(x, weight, eps)
|
||||
x_fp8, scales = rocm_aiter_ops.group_fp8_quant(normed.bfloat16(), group_size)
|
||||
x_dequant = _dequantize_grouped(x_fp8, scales, group_size)
|
||||
|
||||
_assert_rel_error_quality(
|
||||
x_dequant,
|
||||
ref_normed_f32,
|
||||
label=f"rms_norm_then_group_fp8_quant_e2e dtype={dtype}",
|
||||
mean_limit=0.1,
|
||||
preferred_rel=0.1,
|
||||
max_rel=1.0,
|
||||
max_fail_rate=0.01,
|
||||
)
|
||||
@@ -1,226 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm platform and extension smoke tests.
|
||||
|
||||
These tests catch ROCm-specific regressions early:
|
||||
|
||||
1. **Extension import smoke tests**: Verify that ROCm native extensions load
|
||||
without crashing. Build/linking issues surface here before runtime.
|
||||
|
||||
2. **GCN arch parsing tests**: Verify _capability_from_gcn_arch correctly
|
||||
parses all known GPU architectures. Wrong parsing causes silent feature
|
||||
gate failures or crashes from selecting unsupported kernels.
|
||||
|
||||
3. **AITER ops availability tests**: Verify rocm_aiter_ops accessor methods
|
||||
work without crashing, regardless of hardware support.
|
||||
|
||||
4. **Env var propagation tests**: Verify ROCm-specific env vars correctly
|
||||
gate their respective features (e.g., shuffle KV cache, skinny GEMM).
|
||||
|
||||
Related coverage:
|
||||
- AITER kernel numerics: ``tests/kernels/core/test_rocm_aiter_ops.py``
|
||||
- Attention backend selection: ``tests/v1/attention/test_rocm_attention_backends_selection.py``
|
||||
- GEMM dispatch: ``tests/model_executor/layers/test_rocm_unquantized_gemm.py``
|
||||
""" # noqa: E501
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extension import smoke tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rocm_extension_imports():
|
||||
"""ROCm native extension loads without error.
|
||||
|
||||
Catches build/linking issues before runtime. If this fails, the ROCm
|
||||
build is broken and nothing else will work.
|
||||
"""
|
||||
import vllm._rocm_C # noqa: F401
|
||||
|
||||
|
||||
def test_aiter_ops_module_imports():
|
||||
"""AITER ops module imports without error.
|
||||
|
||||
The _aiter_ops module registers custom ops and initializes AITER state.
|
||||
Import failures here indicate missing dependencies or incompatible
|
||||
AITER versions.
|
||||
"""
|
||||
import vllm._aiter_ops # noqa: F401
|
||||
|
||||
|
||||
def test_rocm_platform_module_imports():
|
||||
"""ROCm platform module imports without error.
|
||||
|
||||
The platform module queries hardware via amdsmi and sets up arch flags.
|
||||
Import failures indicate missing amdsmi or HIP runtime issues.
|
||||
"""
|
||||
import vllm.platforms.rocm # noqa: F401
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GCN arch parsing tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("gcn_arch", "expected_major", "expected_minor"),
|
||||
[
|
||||
pytest.param("gfx90a", 9, 0, id="MI200-gfx90a"),
|
||||
pytest.param("gfx942", 9, 4, id="MI300X-gfx942"),
|
||||
pytest.param("gfx950", 9, 5, id="MI350-gfx950"),
|
||||
pytest.param("gfx1100", 11, 0, id="RX7900-gfx1100"),
|
||||
pytest.param("gfx1101", 11, 0, id="RX7800-gfx1101"),
|
||||
pytest.param("gfx1151", 11, 5, id="StrixHalo-gfx1151"),
|
||||
pytest.param("gfx1200", 12, 0, id="RDNA4-gfx1200"),
|
||||
pytest.param("gfx1201", 12, 0, id="RX9070-gfx1201"),
|
||||
pytest.param("gfx1250", 12, 5, id="CDNA5-gfx1250"),
|
||||
],
|
||||
)
|
||||
def test_capability_from_gcn_arch_known_gpus(gcn_arch, expected_major, expected_minor):
|
||||
"""_capability_from_gcn_arch parses known GPU architectures correctly.
|
||||
|
||||
This is critical for feature gating - wrong parsing causes:
|
||||
- FP8 kernels selected on hardware that doesn't support them
|
||||
- MX format support incorrectly enabled/disabled
|
||||
- Custom allreduce used on unsupported hardware
|
||||
"""
|
||||
from vllm.platforms.rocm import _capability_from_gcn_arch
|
||||
|
||||
result = _capability_from_gcn_arch(gcn_arch)
|
||||
assert result is not None, f"Failed to parse {gcn_arch}"
|
||||
assert result == (expected_major, expected_minor), (
|
||||
f"Wrong capability for {gcn_arch}: got {result}, "
|
||||
f"expected ({expected_major}, {expected_minor})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gcn_arch",
|
||||
[
|
||||
pytest.param("not_a_gfx_string", id="non-gfx-string"),
|
||||
pytest.param("cuda_arch", id="cuda-style"),
|
||||
pytest.param("", id="empty-string"),
|
||||
],
|
||||
)
|
||||
def test_capability_from_gcn_arch_returns_none_for_non_gfx(gcn_arch):
|
||||
"""_capability_from_gcn_arch returns None for non-GFX strings.
|
||||
|
||||
Non-GFX strings should return None so the caller can fall back to
|
||||
torch.cuda.get_device_capability.
|
||||
"""
|
||||
from vllm.platforms.rocm import _capability_from_gcn_arch
|
||||
|
||||
result = _capability_from_gcn_arch(gcn_arch)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"gcn_arch",
|
||||
[
|
||||
pytest.param("gfx8", id="too-few-digits"),
|
||||
pytest.param("gfx12345", id="too-many-digits"),
|
||||
pytest.param("gfx700", id="unsupported-major-7"),
|
||||
pytest.param("gfx1500", id="future-major-15"),
|
||||
],
|
||||
)
|
||||
def test_capability_from_gcn_arch_raises_for_malformed(gcn_arch):
|
||||
"""_capability_from_gcn_arch raises ValueError for malformed GFX strings.
|
||||
|
||||
Malformed strings that look like GFX but can't be parsed should raise
|
||||
so the user gets a clear error message asking them to file an issue.
|
||||
"""
|
||||
from vllm.platforms.rocm import _capability_from_gcn_arch
|
||||
|
||||
with pytest.raises(ValueError, match="GCN arch"):
|
||||
_capability_from_gcn_arch(gcn_arch)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AITER ops availability tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rocm_aiter_ops_accessor_methods_dont_crash():
|
||||
"""rocm_aiter_ops accessor methods work without crashing.
|
||||
|
||||
These methods gate kernel selection. Even if AITER isn't available,
|
||||
they should return False gracefully rather than crashing.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# These should all return bool without crashing
|
||||
assert isinstance(rocm_aiter_ops.is_enabled(), (bool, type(None)))
|
||||
assert isinstance(rocm_aiter_ops.is_linear_enabled(), (bool, type(None)))
|
||||
assert isinstance(rocm_aiter_ops.is_fused_moe_enabled(), (bool, type(None)))
|
||||
assert isinstance(rocm_aiter_ops.is_mla_enabled(), (bool, type(None)))
|
||||
assert isinstance(rocm_aiter_ops.is_mha_enabled(), (bool, type(None)))
|
||||
|
||||
|
||||
def test_rocm_aiter_ops_is_aiter_found():
|
||||
"""is_aiter_found returns consistent results."""
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found
|
||||
|
||||
# The cached global should match the function result
|
||||
assert is_aiter_found() == IS_AITER_FOUND
|
||||
|
||||
|
||||
def test_rocm_aiter_ops_refresh_env_variables_doesnt_crash():
|
||||
"""refresh_env_variables works without crashing.
|
||||
|
||||
This is called after monkeypatching env vars in tests. It should
|
||||
never crash, even with unusual env var combinations.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# Should not raise
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Env var propagation tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [True, False])
|
||||
def test_shuffle_kv_cache_env_propagates_to_rocm_aiter_ops(enabled, monkeypatch):
|
||||
"""VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT propagates to AITER state."""
|
||||
import vllm.envs as envs
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", "1" if enabled else "0")
|
||||
importlib.reload(envs)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert envs.VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT is enabled
|
||||
assert rocm_aiter_ops.is_shuffle_kv_cache_enabled() is enabled
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enabled", [True, False])
|
||||
def test_skinny_gemm_env_controls_rocm_fp8_scaled_mm_support(enabled, monkeypatch):
|
||||
"""VLLM_ROCM_USE_SKINNY_GEMM gates ROCm FP8 scaled-mm kernel eligibility."""
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.rocm import (
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "1" if enabled else "0")
|
||||
importlib.reload(envs)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_mi3xx", lambda: True)
|
||||
|
||||
supported, reason = ROCmFP8ScaledMMLinearKernel.is_supported()
|
||||
|
||||
assert supported is enabled
|
||||
if enabled:
|
||||
assert reason is None
|
||||
else:
|
||||
assert reason == "requires VLLM_ROCM_USE_SKINNY_GEMM to be enabled."
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for miscellaneous utilities."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
|
||||
|
||||
def rotary_embedding_opcheck(
|
||||
rot,
|
||||
positions: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor | None = None,
|
||||
):
|
||||
cos_sin_cache = rot.cos_sin_cache.to(query.device, dtype=query.dtype)
|
||||
|
||||
# ops.rotary_embedding() is a in-place operation
|
||||
# that updates the query and key tensors.
|
||||
opcheck(
|
||||
torch.ops._C.rotary_embedding,
|
||||
(positions, query, key, rot.head_size, cos_sin_cache, rot.is_neox_style),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cuda"])
|
||||
@pytest.mark.parametrize("max_position", [11, 4096, 32768])
|
||||
@pytest.mark.parametrize("is_neox_style", [True, False])
|
||||
@pytest.mark.parametrize("rotary_dim", [32])
|
||||
@pytest.mark.parametrize("head_size", [32, 108])
|
||||
@pytest.mark.parametrize("seq_len", [11, 1024])
|
||||
@pytest.mark.parametrize("use_key", [True, False])
|
||||
@pytest.mark.parametrize("head_stride_is_contiguous", [True, False])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_rotary_embedding_opcheck(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
device,
|
||||
max_position,
|
||||
is_neox_style,
|
||||
rotary_dim,
|
||||
head_size,
|
||||
seq_len,
|
||||
use_key,
|
||||
head_stride_is_contiguous,
|
||||
dtype,
|
||||
):
|
||||
batch_size = 1
|
||||
base = 10000
|
||||
num_heads = 7
|
||||
rot = RotaryEmbedding(
|
||||
head_size, rotary_dim, max_position, base, is_neox_style, dtype
|
||||
)
|
||||
|
||||
positions = torch.randint(0, max_position, (batch_size, seq_len), device=device)
|
||||
head_stride = head_size + (64 if head_stride_is_contiguous else 0)
|
||||
|
||||
query = torch.randn(
|
||||
batch_size, seq_len, num_heads, head_stride, dtype=dtype, device=device
|
||||
)
|
||||
key = torch.randn_like(query) if use_key else None
|
||||
query = query[..., :head_size]
|
||||
key = key[..., :head_size] if key is not None else None
|
||||
|
||||
rotary_embedding_opcheck(rot, positions, query, key)
|
||||
|
||||
# if we have a contiguous head stride, test the alternate
|
||||
# [..., num_heads * head_dim] shape/layout
|
||||
if head_stride_is_contiguous:
|
||||
rotary_embedding_opcheck(
|
||||
rot,
|
||||
positions,
|
||||
query.flatten(start_dim=-2),
|
||||
key.flatten(start_dim=-2) if key is not None else None,
|
||||
)
|
||||
@@ -1,325 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for fused MLA KV-cache write and RoPE fused kernel."""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
def assert_rope_close_rocm(
|
||||
actual: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
rope_input: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
is_neox: bool,
|
||||
*,
|
||||
max_fail_fraction: float | None = None,
|
||||
atol: float | None = None,
|
||||
rtol: float | None = None,
|
||||
) -> None:
|
||||
"""assert_close for RoPE fused-vs-reference on ROCm, with a per-element
|
||||
product-ULP allowance for catastrophic-cancellation elements.
|
||||
|
||||
The fused and reference kernels may contract the RoPE multiply-add into an
|
||||
FMA with either product pre-rounded to the working dtype. The two valid
|
||||
orderings differ by at most 2 ULPs of the product at that element, which
|
||||
cancellation (x*cos ~= -y*sin) can amplify past the default tolerance.
|
||||
|
||||
Tier 1: |actual - ref| <= atol + rtol * |ref| (default tolerance).
|
||||
Tier 2: elements failing tier 1 must each satisfy
|
||||
|actual_e - ref_e| <= 2 * eps * |product_e|, where product_e is the larger
|
||||
of the two products combined at that element, and their fraction must be
|
||||
<= max_fail_fraction (1e-4 for fp16; 1e-2 for bf16, whose coarser eps
|
||||
puts 1-ULP divergences past the default tolerance far more often).
|
||||
"""
|
||||
if atol is None:
|
||||
atol = get_default_atol(ref)
|
||||
if rtol is None:
|
||||
rtol = get_default_rtol(ref)
|
||||
if max_fail_fraction is None:
|
||||
max_fail_fraction = 1e-2 if ref.dtype == torch.bfloat16 else 1e-4
|
||||
eps = torch.finfo(ref.dtype).eps
|
||||
|
||||
# per-element product magnitude, in the (seq, heads, rot_dim) layout
|
||||
seq_len, rot_dim = rope_input.shape[0], rope_input.shape[-1]
|
||||
half = rot_dim // 2
|
||||
inp = rope_input.reshape(seq_len, -1, rot_dim).to(torch.float32)
|
||||
cs = cos_sin_cache[positions].to(torch.float32) # (seq_len, 2*half)
|
||||
cos = cs[:, :half].unsqueeze(1) # broadcast over heads
|
||||
sin = cs[:, half:].unsqueeze(1)
|
||||
x, y = inp[..., :half], inp[..., half:]
|
||||
if is_neox:
|
||||
# out = [x*cos - y*sin, y*cos + x*sin]
|
||||
prod = torch.cat(
|
||||
[
|
||||
torch.maximum((x * cos).abs(), (y * sin).abs()),
|
||||
torch.maximum((y * cos).abs(), (x * sin).abs()),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
else:
|
||||
# out[2i] = x[2i]*cos[i] - x[2i+1]*sin[i], interleaved
|
||||
xe, xo = inp[..., 0::2], inp[..., 1::2]
|
||||
prod = torch.stack(
|
||||
[
|
||||
torch.maximum((xe * cos).abs(), (xo * sin).abs()),
|
||||
torch.maximum((xo * cos).abs(), (xe * sin).abs()),
|
||||
],
|
||||
dim=-1,
|
||||
).flatten(-2)
|
||||
prod = prod.reshape(rope_input.shape)
|
||||
|
||||
diff = (actual.to(torch.float32) - ref.to(torch.float32)).abs()
|
||||
tol = atol + rtol * ref.to(torch.float32).abs()
|
||||
fail = diff > tol
|
||||
n_fail = int(fail.sum())
|
||||
if n_fail == 0:
|
||||
return
|
||||
assert n_fail <= max_fail_fraction * diff.numel(), (
|
||||
f"{n_fail} elements beyond default tolerance "
|
||||
f"({n_fail / diff.numel():.2%} > {max_fail_fraction:.2%}); "
|
||||
f"max diff {diff.max().item()}"
|
||||
)
|
||||
excess = diff[fail] - 2 * eps * prod[fail]
|
||||
assert (excess <= 0).all(), (
|
||||
f"{int((excess > 0).sum())} cancellation elements exceed 2 product-ULPs; "
|
||||
f"worst excess {excess.max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def default_vllm_config(monkeypatch):
|
||||
"""Enable the AITER triton rope on ROCm for fp16-consistent numerics.
|
||||
|
||||
The fused CUDA kernel runs native fp16 while forward_native upcasts to
|
||||
fp32, so on ROCm we route through the AITER triton rope (+rotary_embedding)
|
||||
to match. Its env gates are cached at import, hence refresh_env_variables().
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
|
||||
is_rocm = current_platform.is_rocm()
|
||||
if is_rocm:
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["+rotary_embedding"])
|
||||
)
|
||||
else:
|
||||
config = VllmConfig()
|
||||
try:
|
||||
with monkeypatch.context() as m, set_current_vllm_config(config):
|
||||
if is_rocm:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv("VLLM_ROCM_USE_AITER_TRITON_ROPE", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
yield config
|
||||
finally:
|
||||
if is_rocm:
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16, torch.float])
|
||||
@pytest.mark.parametrize("is_neox_style", [False, True])
|
||||
@pytest.mark.parametrize("seq_len", [11, 42])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64, 128])
|
||||
@pytest.mark.parametrize("num_q_heads", [128])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("num_blocks", [64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 256])
|
||||
@pytest.mark.parametrize("seed", [0])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_concat_and_cache_mla_rope_fused(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
is_neox_style: bool,
|
||||
seq_len: int,
|
||||
qk_rope_head_dim: int,
|
||||
num_q_heads: int,
|
||||
kv_cache_dtype: str,
|
||||
kv_lora_rank: int,
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
seed: int,
|
||||
device: str,
|
||||
max_position: int = 8192,
|
||||
base: float = 10000,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
|
||||
rope = RotaryEmbedding(
|
||||
qk_rope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
max_position,
|
||||
base,
|
||||
is_neox_style,
|
||||
torch.float32,
|
||||
)
|
||||
|
||||
rope = rope.to(dtype=dtype, device=torch.get_default_device())
|
||||
|
||||
positions = torch.randint(0, max_position, (seq_len,))
|
||||
|
||||
query = torch.randn(seq_len, num_q_heads, qk_rope_head_dim, dtype=dtype)
|
||||
key = torch.randn(seq_len, 1, qk_rope_head_dim + kv_lora_rank, dtype=dtype)
|
||||
|
||||
k_pe = torch.flatten(key[..., :qk_rope_head_dim], start_dim=1).to(device=device)
|
||||
kv_c = torch.flatten(key[..., qk_rope_head_dim:], start_dim=1).to(device=device)
|
||||
k_pe_orig = k_pe.clone()
|
||||
query_orig = query.clone()
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# We use forward_hip for the same numerics as the fused custom kernel on ROCm
|
||||
# when dtype is FP16. The torch-native implementation implicitly upcasts
|
||||
# FP16 x FP16 multiplications to FP32 before downcasting them, which leads
|
||||
# to notable output divergences.
|
||||
# Clone the tensors because the implementation modifies them in-place
|
||||
ref_q_pe, ref_k_pe = rope.forward_hip(positions, query.clone(), k_pe.clone())
|
||||
else:
|
||||
# NOTE(woosuk): The reference implementation should be executed first
|
||||
# because the custom kernel is in-place.
|
||||
ref_q_pe, ref_k_pe = rope.forward_native(positions, query, k_pe)
|
||||
assert ref_k_pe is not None
|
||||
|
||||
ref_k_pe = torch.flatten(ref_k_pe, start_dim=1).to(device=device)
|
||||
ref_k_rope = ref_k_pe[..., :qk_rope_head_dim]
|
||||
|
||||
total_available_slots = num_blocks * block_size
|
||||
total_needed_slots = seq_len
|
||||
assert total_available_slots >= total_needed_slots, "Not enough kv slots!"
|
||||
|
||||
slot_mapping_lst = random.sample(range(total_available_slots), total_needed_slots)
|
||||
slot_mapping = torch.tensor(slot_mapping_lst, dtype=torch.long, device=device)
|
||||
|
||||
entry_size = kv_lora_rank + qk_rope_head_dim
|
||||
|
||||
kv_cache_scale = torch.tensor([0.1], dtype=torch.float32, device=device)
|
||||
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
block_size,
|
||||
entry_size,
|
||||
dtype=torch.uint8 if kv_cache_dtype == "fp8" else dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
ref_temp = torch.zeros(*kv_cache.shape, dtype=dtype, device=device)
|
||||
|
||||
for i in range(seq_len):
|
||||
slot = slot_mapping[i].item()
|
||||
block_idx = slot // block_size
|
||||
block_offset = slot % block_size
|
||||
ref_temp[block_idx, block_offset] = torch.cat((kv_c[i], ref_k_rope[i]), -1)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
ref_kv_cache = torch.empty_like(ref_temp, dtype=kv_cache.dtype)
|
||||
ops.convert_fp8(
|
||||
ref_kv_cache, ref_temp, kv_cache_scale.item(), kv_dtype=kv_cache_dtype
|
||||
)
|
||||
else:
|
||||
ref_kv_cache = ref_temp
|
||||
|
||||
opcheck(
|
||||
torch.ops._C_cache_ops.concat_and_cache_mla_rope_fused,
|
||||
(
|
||||
positions,
|
||||
query,
|
||||
k_pe,
|
||||
kv_c,
|
||||
rope.cos_sin_cache,
|
||||
is_neox_style,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
),
|
||||
test_utils=DEFAULT_OPCHECK_TEST_UTILS,
|
||||
)
|
||||
|
||||
ops.concat_and_cache_mla_rope_fused(
|
||||
positions,
|
||||
query,
|
||||
k_pe,
|
||||
kv_c,
|
||||
rope.cos_sin_cache,
|
||||
is_neox_style,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
kv_cache_dtype,
|
||||
kv_cache_scale,
|
||||
)
|
||||
|
||||
# On ROCm the AITER Triton rope diverges by up to 2 product-ULPs from the
|
||||
# fused kernel, which the default tolerance misses at the rare
|
||||
# catastrophic-cancellation elements. fp8 keeps its calibrated tolerance.
|
||||
rocm_neox = current_platform.is_rocm() and is_neox_style
|
||||
rocm_bf16 = current_platform.is_rocm() and dtype == torch.bfloat16
|
||||
if kv_cache_dtype == "fp8":
|
||||
result_temp = torch.empty_like(kv_cache, dtype=torch.float16)
|
||||
ops.convert_fp8(
|
||||
result_temp,
|
||||
kv_cache.contiguous(),
|
||||
kv_cache_scale.item(),
|
||||
kv_dtype=kv_cache_dtype,
|
||||
)
|
||||
expected_temp = torch.empty_like(ref_kv_cache, dtype=torch.float16)
|
||||
ops.convert_fp8(
|
||||
expected_temp, ref_kv_cache, kv_cache_scale.item(), kv_dtype=kv_cache_dtype
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result_temp,
|
||||
expected_temp,
|
||||
atol=0.004 if rocm_bf16 else 0.001,
|
||||
rtol=0.15 if rocm_neox or rocm_bf16 else 0.1,
|
||||
)
|
||||
elif current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
|
||||
# gather the rope region of the cache back to token order
|
||||
block_idx = slot_mapping // block_size
|
||||
block_offset = slot_mapping % block_size
|
||||
fused_k_rope = kv_cache[block_idx, block_offset, kv_lora_rank:]
|
||||
ref_k_rope_gathered = ref_kv_cache[block_idx, block_offset, kv_lora_rank:]
|
||||
assert_rope_close_rocm(
|
||||
fused_k_rope,
|
||||
ref_k_rope_gathered,
|
||||
k_pe_orig,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
is_neox_style,
|
||||
)
|
||||
# the nope region is a byte-exact copy
|
||||
torch.testing.assert_close(
|
||||
kv_cache[..., :kv_lora_rank], ref_kv_cache[..., :kv_lora_rank]
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(kv_cache, ref_kv_cache)
|
||||
|
||||
if current_platform.is_rocm() and dtype in (torch.half, torch.bfloat16):
|
||||
assert_rope_close_rocm(
|
||||
query,
|
||||
ref_q_pe,
|
||||
query_orig,
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
is_neox_style,
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
query,
|
||||
ref_q_pe,
|
||||
atol=get_default_atol(query),
|
||||
rtol=get_default_rtol(query),
|
||||
)
|
||||
@@ -1,292 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
from vllm.v1.worker.gpu import buffer_utils
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
DEVICES = [
|
||||
f"{DEVICE_TYPE}:{i}"
|
||||
for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_cpu_write(device):
|
||||
torch.set_default_device(device)
|
||||
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
|
||||
# as the UVA view's ops never trigger lazy init.
|
||||
torch.accelerator.set_device_index(device)
|
||||
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
|
||||
assert gpu_view[0, 0] == 0
|
||||
assert gpu_view[2, 3] == 0
|
||||
assert gpu_view[4, 5] == 0
|
||||
|
||||
cpu_tensor[0, 0] = 1
|
||||
cpu_tensor[2, 3] = 2
|
||||
cpu_tensor[4, 5] = -1
|
||||
|
||||
gpu_view.mul_(2)
|
||||
torch.accelerator.synchronize(gpu_view.device)
|
||||
assert gpu_view[0, 0] == 2
|
||||
assert gpu_view[2, 3] == 4
|
||||
assert gpu_view[4, 5] == -2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_gpu_write(device):
|
||||
torch.set_default_device(device)
|
||||
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
|
||||
# as the UVA view's ops never trigger lazy init.
|
||||
torch.accelerator.set_device_index(device)
|
||||
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
|
||||
assert gpu_view[0, 0] == 0
|
||||
assert gpu_view[2, 3] == 0
|
||||
assert gpu_view[4, 5] == 0
|
||||
|
||||
gpu_view[0, 0] = 1
|
||||
gpu_view[2, 3] = 2
|
||||
gpu_view[4, 5] = -1
|
||||
gpu_view.mul_(2)
|
||||
torch.accelerator.synchronize(gpu_view.device)
|
||||
|
||||
assert cpu_tensor[0, 0] == 2
|
||||
assert cpu_tensor[2, 3] == 4
|
||||
assert cpu_tensor[4, 5] == -2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_staged_write_uses_uva_contents_for_uva_target(device, monkeypatch):
|
||||
def fail_async_tensor_h2d(*args, **kwargs):
|
||||
pytest.fail("UVA-backed targets should not copy write contents to the GPU")
|
||||
|
||||
monkeypatch.setattr(buffer_utils, "async_tensor_h2d", fail_async_tensor_h2d)
|
||||
staged = StagedWriteTensor(
|
||||
(3, 4096),
|
||||
dtype=torch.int32,
|
||||
device=torch.device(device),
|
||||
max_concurrency=2,
|
||||
uva_instead_of_gpu=True,
|
||||
)
|
||||
|
||||
staged.stage_write(2, 3, [11, 12, 13])
|
||||
staged.apply_write()
|
||||
torch.accelerator.synchronize()
|
||||
staged.stage_write(1, 7, [21, 22])
|
||||
staged.apply_write()
|
||||
torch.accelerator.synchronize()
|
||||
staged.stage_write(0, 1020, range(1500))
|
||||
staged.apply_write()
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert staged.gpu[2, 3:6].tolist() == [11, 12, 13]
|
||||
assert staged.gpu[1, 7:9].tolist() == [21, 22]
|
||||
assert staged.gpu[0, 1020:2520].tolist() == list(range(1500))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("input_type", ["list", "numpy", "tensor"])
|
||||
@pytest.mark.parametrize("size", [(4,), (4, 3)])
|
||||
def test_uva_pool_overwrites_exposed_prefix(input_type, size):
|
||||
"""Both slots expose only current contents across growth and shorter reuse."""
|
||||
pool = buffer_utils.UvaBufferPool(size, torch.int32, max_concurrency=2)
|
||||
for buf in pool._uva_bufs:
|
||||
assert tuple(buf.cpu.shape) == size
|
||||
assert buf.cpu.is_pinned()
|
||||
assert torch.count_nonzero(buf.cpu).item() == 0
|
||||
lengths = [0, 0, 3, 3, 4, 4, 5, 3, 3, 5, 1024, 1024, 1025, 1025, 2, 2]
|
||||
for step, length in enumerate(lengths):
|
||||
if input_type == "list" and len(size) > 1 and length == 0:
|
||||
# An empty list has no trailing shape; preserve NumPy's rejection.
|
||||
with pytest.raises(ValueError, match="could not broadcast"):
|
||||
pool.copy_to_uva([])
|
||||
continue
|
||||
shape = (length, *size[1:])
|
||||
expected = (
|
||||
torch.arange(int(np.prod(shape)), dtype=torch.int32, device="cpu").reshape(
|
||||
shape
|
||||
)
|
||||
- step
|
||||
)
|
||||
values = expected.tolist()
|
||||
if input_type == "numpy":
|
||||
values = expected.numpy()
|
||||
elif input_type == "tensor":
|
||||
values = expected
|
||||
before = list(pool._uva_bufs)
|
||||
slot = (pool._curr + 1) % pool.max_concurrency
|
||||
result = pool.copy_to_uva(values)
|
||||
assert tuple(result.shape) == shape
|
||||
assert pool._curr == slot
|
||||
assert pool._uva_bufs[1 - slot] is before[1 - slot]
|
||||
if length <= before[slot].cpu.shape[0]:
|
||||
assert pool._uva_bufs[slot] is before[slot]
|
||||
else:
|
||||
assert tuple(pool._uva_bufs[slot].cpu.shape) == (
|
||||
1 << (length - 1).bit_length(),
|
||||
*size[1:],
|
||||
)
|
||||
assert pool.size == size
|
||||
# Blocking read also retires GPU readers before the next slot reuse.
|
||||
torch.testing.assert_close(result.cpu(), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("use_out", [False, True])
|
||||
@pytest.mark.parametrize("input_type", ["numpy", "tensor"])
|
||||
def test_uva_pool_copy_to_gpu_preserves_shape_and_out(use_out, input_type):
|
||||
pool = buffer_utils.UvaBufferPool((2, 3), torch.int32, max_concurrency=2)
|
||||
for length in (2, 5, 3, 6):
|
||||
expected = torch.arange(length * 3, dtype=torch.int32, device="cpu").reshape(
|
||||
length, 3
|
||||
)
|
||||
values = expected.numpy() if input_type == "numpy" else expected
|
||||
out = (
|
||||
torch.empty(expected.shape, dtype=torch.int32, device=DEVICE_TYPE)
|
||||
if use_out
|
||||
else None
|
||||
)
|
||||
result = pool.copy_to_gpu(values, out=out)
|
||||
if use_out:
|
||||
assert result is out
|
||||
torch.testing.assert_close(result.cpu(), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("uva_target", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.int32, torch.int64, torch.float32])
|
||||
def test_staged_write_inflight(uva_target, dtype):
|
||||
"""Preserve every generation until its consumer finishes before slot reuse."""
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
with torch.accelerator.device_index(device.index):
|
||||
state = StagedWriteTensor(
|
||||
(4, 4096),
|
||||
dtype,
|
||||
device,
|
||||
max_concurrency=2,
|
||||
uva_instead_of_gpu=uva_target,
|
||||
)
|
||||
assert (state.write_contents is not None) == uva_target
|
||||
stream = torch.Stream(device=device)
|
||||
stream.wait_stream(torch.accelerator.current_stream())
|
||||
pending: list[tuple[torch.Event, torch.Tensor, torch.Tensor]] = []
|
||||
expected = torch.zeros((4, 4096), dtype=dtype, device="cpu")
|
||||
for step in range(24):
|
||||
if len(pending) == 2:
|
||||
event, snapshot, reference = pending.pop(0)
|
||||
event.synchronize()
|
||||
torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0)
|
||||
# Growing and shrinking lengths exercise reallocation and reuse.
|
||||
length = [3, 17, 1500, 4090][step % 4]
|
||||
row = step % 3
|
||||
values = torch.arange(length, dtype=dtype, device="cpu") + step * 8192
|
||||
if dtype == torch.float32:
|
||||
values += 0.25
|
||||
expected[row, 2 : 2 + length] = values
|
||||
expected[3, 1:4] = step
|
||||
with stream:
|
||||
state.stage_write(row, 2, values.tolist())
|
||||
state.stage_write(3, 1, [step] * 3)
|
||||
state.apply_write()
|
||||
# A GPU consumer observes this generation before the next update.
|
||||
snapshot = state.gpu.clone()
|
||||
event = torch.Event()
|
||||
event.record(stream)
|
||||
pending.append((event, snapshot, expected.clone()))
|
||||
for event, snapshot, reference in pending:
|
||||
event.synchronize()
|
||||
torch.testing.assert_close(snapshot.cpu(), reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_non_pinned_cpu_tensor(device):
|
||||
# Non-pinned CPU tensors are internally copied into a pinned buffer,
|
||||
# so the resulting gpu view reflects the values at creation time but
|
||||
# is decoupled from further writes to the original `cpu_tensor`.
|
||||
torch.set_default_device(device)
|
||||
cpu_tensor = torch.arange(100, dtype=torch.int32, device="cpu").view(10, 10)
|
||||
assert not cpu_tensor.is_pinned()
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
|
||||
assert gpu_view[0, 0] == 0
|
||||
assert gpu_view[2, 3] == 23
|
||||
assert gpu_view[9, 9] == 99
|
||||
|
||||
# Writes to the original (unpinned) CPU tensor must not affect the view,
|
||||
# since a private pinned copy was made.
|
||||
cpu_tensor[0, 0] = -1
|
||||
assert gpu_view[0, 0] == 0
|
||||
|
||||
# The view itself remains writable and independently usable.
|
||||
gpu_view.mul_(2)
|
||||
assert gpu_view[2, 3] == 46
|
||||
assert gpu_view[9, 9] == 198
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only sync check.")
|
||||
def test_non_uva_buffer_copies_without_stream_sync():
|
||||
buffer = buffer_utils.NonUvaBuffer(8, torch.int32)
|
||||
buffer.np[:] = np.arange(8)
|
||||
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
prefix = buffer.uva(4)
|
||||
full = buffer.uva()
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(0)
|
||||
# The next step overwrites the host buffer right after the copy is issued.
|
||||
buffer.np[:] = -1
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert prefix.tolist() == list(range(4))
|
||||
assert full.tolist() == list(range(8))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_xpu(), reason="XPU non-contiguous UVA test."
|
||||
)
|
||||
@pytest.mark.parametrize("pinned", [False, True])
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_non_contiguous_strided_view(device, pinned):
|
||||
torch.set_default_device(device)
|
||||
# Simulate scale_kn: a [32, 16] contiguous tensor
|
||||
scale_kn = torch.arange(
|
||||
512, dtype=torch.float32, device="cpu", pin_memory=pinned
|
||||
).view(32, 16)
|
||||
# Transposed view: shape [16, 32], non-contiguous stride (1, 16)
|
||||
cpu_view = scale_kn.t()
|
||||
assert cpu_view.shape == (16, 32)
|
||||
assert cpu_view.stride() == (1, 16)
|
||||
assert not cpu_view.is_contiguous()
|
||||
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_view)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
assert gpu_view.shape == (16, 32)
|
||||
assert gpu_view.stride() == (1, 16)
|
||||
assert not gpu_view.is_contiguous()
|
||||
|
||||
# Transposing tensor_view back should yield a contiguous [32, 16] tensor
|
||||
gpu_view_t = gpu_view.t()
|
||||
assert gpu_view_t.shape == (32, 16)
|
||||
assert gpu_view_t.is_contiguous()
|
||||
|
||||
# Correctness check: values match original transposed CPU tensor
|
||||
assert torch.equal(gpu_view.cpu(), cpu_view)
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the stride-aware FP8 quantization kernel with head_dim padding."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
if HAS_TRITON:
|
||||
from vllm.kernels.triton.qkv_padded_fp8_quant import (
|
||||
quantize_fp8_pad_head_dim_triton,
|
||||
)
|
||||
|
||||
HEAD_DIMS = [72, 80, 128]
|
||||
SEQ_LENS = [64, 256]
|
||||
NUM_HEADS = [16]
|
||||
SCALES = [0.01, 0.1, 1.0]
|
||||
|
||||
|
||||
def _naive_fp8_quantize(
|
||||
tensor: torch.Tensor, scale: torch.Tensor, skip_scale: bool
|
||||
) -> torch.Tensor:
|
||||
"""Reference FP8 quantization in PyTorch."""
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
|
||||
x = tensor.float()
|
||||
if not skip_scale:
|
||||
x = x / scale.item()
|
||||
x = x.clamp(fp8_min, fp8_max)
|
||||
return x.to(fp8_dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
@pytest.mark.parametrize("head_dim", HEAD_DIMS)
|
||||
@pytest.mark.parametrize("seq_len", SEQ_LENS)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("scale_val", SCALES)
|
||||
def test_quantize_contiguous(
|
||||
head_dim: int, seq_len: int, num_heads: int, scale_val: float
|
||||
) -> None:
|
||||
"""Test quantization of contiguous 3D tensors."""
|
||||
torch.manual_seed(42)
|
||||
tensor = torch.randn(
|
||||
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
scale = torch.tensor([scale_val], dtype=torch.float32, device="cuda").view(
|
||||
1, 1, 1, 1
|
||||
)
|
||||
|
||||
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
|
||||
|
||||
padded_dim = (head_dim + 15) // 16 * 16
|
||||
assert result.shape == (seq_len, num_heads, padded_dim)
|
||||
assert result.is_contiguous()
|
||||
assert result.dtype == current_platform.fp8_dtype()
|
||||
|
||||
# Compare unpadded portion against reference
|
||||
ref = _naive_fp8_quantize(tensor, scale, skip_scale=False)
|
||||
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
|
||||
|
||||
# Padded region should be zero
|
||||
if padded_dim > head_dim:
|
||||
assert (result[:, :, head_dim:].float() == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
@pytest.mark.parametrize("head_dim", [72, 80])
|
||||
def test_quantize_non_contiguous(head_dim: int) -> None:
|
||||
"""Test quantization from non-contiguous QKV views (interleaved buffer)."""
|
||||
seq_len, num_heads = 64, 16
|
||||
# Simulate interleaved QKV buffer: shape (seq_len, 3 * num_heads, head_dim)
|
||||
qkv = torch.randn(
|
||||
seq_len, 3 * num_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
# Q is every 3rd head slice - non-contiguous view
|
||||
q = qkv[:, 0::3, :]
|
||||
assert not q.is_contiguous()
|
||||
|
||||
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
|
||||
result = quantize_fp8_pad_head_dim_triton(q, scale)
|
||||
|
||||
padded_dim = (head_dim + 15) // 16 * 16
|
||||
assert result.shape == (seq_len, num_heads, padded_dim)
|
||||
assert result.is_contiguous()
|
||||
|
||||
# Compare against contiguous reference
|
||||
ref = _naive_fp8_quantize(q.contiguous(), scale, skip_scale=False)
|
||||
torch.testing.assert_close(result[:, :, :head_dim].float(), ref.float())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
def test_skip_scale() -> None:
|
||||
"""Test skip_scale=True produces cast-only output (no division)."""
|
||||
seq_len, num_heads, head_dim = 32, 8, 80
|
||||
tensor = torch.randn(
|
||||
seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
|
||||
|
||||
result_skip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=True)
|
||||
result_noskip = quantize_fp8_pad_head_dim_triton(tensor, scale, skip_scale=False)
|
||||
|
||||
# skip_scale should just cast, not divide
|
||||
ref_cast = _naive_fp8_quantize(tensor, scale, skip_scale=True)
|
||||
torch.testing.assert_close(result_skip[:, :, :head_dim].float(), ref_cast.float())
|
||||
|
||||
# With scale != 1.0, skip and no-skip should differ
|
||||
assert not torch.equal(result_skip.float(), result_noskip.float())
|
||||
|
||||
|
||||
@pytest.mark.skipif(not HAS_TRITON, reason="Triton not available")
|
||||
def test_4d_input() -> None:
|
||||
"""Test that 4D input (B, S, H, D) is handled correctly."""
|
||||
B, S, H, D = 2, 32, 8, 72
|
||||
tensor = torch.randn(B, S, H, D, device="cuda", dtype=torch.bfloat16)
|
||||
scale = torch.tensor([0.1], dtype=torch.float32, device="cuda").view(1, 1, 1, 1)
|
||||
|
||||
result = quantize_fp8_pad_head_dim_triton(tensor, scale)
|
||||
padded_dim = (D + 15) // 16 * 16
|
||||
assert result.shape == (B, S, H, padded_dim)
|
||||
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for FP8 scaling (dynamic and static) in MMEncoderAttention."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
_FP8_AMAX_HISTORY_LEN,
|
||||
_FP8_MAX,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
is_flashinfer_cudnn_fp8_prefill_attn_supported,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not is_flashinfer_cudnn_fp8_prefill_attn_supported(),
|
||||
reason="FlashInfer cuDNN FP8 prefill attention not supported",
|
||||
)
|
||||
|
||||
LAYER_0 = "visual.blocks.0.attn.attn"
|
||||
LAYER_1 = "visual.blocks.1.attn.attn"
|
||||
NUM_HEADS = 16
|
||||
HEAD_DIM = 72
|
||||
|
||||
|
||||
def _is_mi3xx() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return on_mi3xx()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _build_attention(mm_config, attn_backend=None):
|
||||
"""Yield an MMEncoderAttention with the given multimodal config.
|
||||
|
||||
The VllmConfig context stays active while the test runs so that
|
||||
``get_multimodal_config()`` calls during the forward path resolve. Also
|
||||
invokes ``process_weights_after_loading`` to simulate the model loader's
|
||||
auto-scan. Yields ``None`` if FlashInfer cuDNN is not available.
|
||||
"""
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
if attn_backend is None:
|
||||
attn_backend = AttentionBackendEnum.FLASHINFER
|
||||
|
||||
if (
|
||||
attn_backend == AttentionBackendEnum.FLASHINFER
|
||||
and not is_flashinfer_cudnn_fp8_prefill_attn_supported()
|
||||
):
|
||||
yield None
|
||||
return
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = MagicMock(spec=ModelConfig, multimodal_config=mm_config)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.model_executor.layers.attention.mm_encoder_attention"
|
||||
".get_vit_attn_backend",
|
||||
return_value=attn_backend,
|
||||
),
|
||||
):
|
||||
attn = MMEncoderAttention(
|
||||
num_heads=NUM_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
prefix=LAYER_0,
|
||||
)
|
||||
attn.process_weights_after_loading(torch.bfloat16)
|
||||
yield attn
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _make_attention():
|
||||
"""Create an MMEncoderAttention with dynamic FP8 scaling."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
with _build_attention(MultiModalConfig(mm_encoder_attn_dtype="fp8")) as attn:
|
||||
yield attn
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _make_static_attention(tmp_path):
|
||||
"""Create an MMEncoderAttention with static FP8 scales from a file."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
scale_file = tmp_path / "scales.json"
|
||||
scale_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0},
|
||||
LAYER_1: {"q": 100.0, "k": 110.0, "v": 120.0},
|
||||
}
|
||||
)
|
||||
)
|
||||
with _build_attention(
|
||||
MultiModalConfig(
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
mm_encoder_fp8_scale_path=str(scale_file),
|
||||
)
|
||||
) as attn:
|
||||
yield attn
|
||||
|
||||
|
||||
def test_dynamic_scaling_updates_scales(_make_attention) -> None:
|
||||
"""Verify that _record_amax_and_update_scales updates scale buffers."""
|
||||
attn = _make_attention
|
||||
if attn is None or not attn.fp8_enabled:
|
||||
pytest.skip("FP8 attention not available (FlashInfer backend required)")
|
||||
|
||||
attn = attn.to("cuda")
|
||||
|
||||
S, H, D = 32, NUM_HEADS, HEAD_DIM
|
||||
q = torch.full((S, H, D), 2.0, device="cuda", dtype=torch.bfloat16)
|
||||
k = torch.full((S, H, D), 3.0, device="cuda", dtype=torch.bfloat16)
|
||||
v = torch.full((S, H, D), 4.0, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
attn._record_amax_and_update_scales(q, k, v)
|
||||
|
||||
expected_q_scale = 2.0 / _FP8_MAX
|
||||
expected_k_scale = 3.0 / _FP8_MAX
|
||||
expected_v_scale = 4.0 / _FP8_MAX
|
||||
|
||||
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_q_scale)
|
||||
torch.testing.assert_close(attn._fp8_k_scale.item(), expected_k_scale)
|
||||
torch.testing.assert_close(attn._fp8_v_scale.item(), expected_v_scale)
|
||||
|
||||
|
||||
def test_circular_buffer_wraps(_make_attention) -> None:
|
||||
"""Verify the amax circular buffer wraps at HISTORY_LEN."""
|
||||
attn = _make_attention
|
||||
if attn is None or not attn.fp8_enabled:
|
||||
pytest.skip("FP8 attention not available (FlashInfer backend required)")
|
||||
|
||||
attn = attn.to("cuda")
|
||||
S, H, D = 16, NUM_HEADS, HEAD_DIM
|
||||
|
||||
for i in range(_FP8_AMAX_HISTORY_LEN + 2):
|
||||
mag = float(i + 1)
|
||||
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
|
||||
k = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
|
||||
v = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
|
||||
attn._record_amax_and_update_scales(q, k, v)
|
||||
|
||||
assert attn._fp8_amax_pos == 2
|
||||
|
||||
expected_max = float(_FP8_AMAX_HISTORY_LEN + 2)
|
||||
expected_scale = expected_max / _FP8_MAX
|
||||
torch.testing.assert_close(attn._fp8_q_scale.item(), expected_scale)
|
||||
|
||||
|
||||
def test_static_scales_loaded(_make_static_attention) -> None:
|
||||
"""Verify static scales are loaded from the JSON file."""
|
||||
attn = _make_static_attention
|
||||
if attn is None or not attn.fp8_enabled:
|
||||
pytest.skip("FP8 attention not available (FlashInfer backend required)")
|
||||
|
||||
assert attn.fp8_enabled
|
||||
assert not attn._fp8_dynamic_scale
|
||||
|
||||
# Layer 0 scales (the layer this attention was created with).
|
||||
assert attn._fp8_q_scale.item() == 224.0
|
||||
assert attn._fp8_k_scale.item() == 198.0
|
||||
assert attn._fp8_v_scale.item() == 210.0
|
||||
|
||||
assert not attn.skip_scale_q
|
||||
assert not attn.skip_scale_k
|
||||
assert not attn.skip_scale_v
|
||||
|
||||
# No amax history buffers for static scaling.
|
||||
assert not hasattr(attn, "_fp8_q_amax")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _is_mi3xx(),
|
||||
reason="AITER FP8 attention requires MI300/MI350",
|
||||
)
|
||||
def test_aiter_static_scales_loaded(tmp_path) -> None:
|
||||
"""Verify AITER reuses the existing static FP8 scale loading path."""
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
scale_file = tmp_path / "aiter_scales.json"
|
||||
scale_file.write_text(json.dumps({LAYER_0: {"q": 224.0, "k": 198.0, "v": 210.0}}))
|
||||
mm_config = MultiModalConfig(
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
mm_encoder_fp8_scale_path=str(scale_file),
|
||||
)
|
||||
|
||||
with _build_attention(mm_config, AttentionBackendEnum.ROCM_AITER_FA) as attn:
|
||||
assert attn is not None
|
||||
assert attn.fp8_enabled
|
||||
assert not attn._fp8_dynamic_scale
|
||||
assert attn._fp8_q_scale.item() == 224.0
|
||||
assert attn._fp8_k_scale.item() == 198.0
|
||||
assert attn._fp8_v_scale.item() == 210.0
|
||||
|
||||
|
||||
def test_static_scales_missing_layer(tmp_path) -> None:
|
||||
"""Verify error when requested layer is not in the scale file."""
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
pytest.skip("FlashInfer cuDNN not available")
|
||||
|
||||
scale_file = tmp_path / "wrong_layer.json"
|
||||
scale_file.write_text(
|
||||
json.dumps({"visual.blocks.99.attn": {"q": 1.0, "k": 1.0, "v": 1.0}})
|
||||
)
|
||||
mm_config = MultiModalConfig(
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
mm_encoder_fp8_scale_path=str(scale_file),
|
||||
)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = MagicMock(spec=ModelConfig, multimodal_config=mm_config)
|
||||
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.model_executor.layers.attention.mm_encoder_attention"
|
||||
".get_vit_attn_backend",
|
||||
return_value=AttentionBackendEnum.FLASHINFER,
|
||||
),
|
||||
):
|
||||
attn = MMEncoderAttention(
|
||||
num_heads=NUM_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
prefix=LAYER_0,
|
||||
)
|
||||
with pytest.raises(ValueError, match="scales not found for layer"):
|
||||
attn.process_weights_after_loading(torch.bfloat16)
|
||||
|
||||
|
||||
def test_dynamic_scales_auto_save(tmp_path) -> None:
|
||||
"""Verify scales are saved to disk after the amax buffer fills."""
|
||||
import vllm.model_executor.layers.attention.mm_encoder_attention as _mod
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
if not is_flashinfer_cudnn_fp8_prefill_attn_supported():
|
||||
pytest.skip("FlashInfer cuDNN not available")
|
||||
|
||||
# Reset module-level state between runs (other tests may have left
|
||||
# state behind after triggering a save).
|
||||
_mod._fp8_scale_save_path = None
|
||||
_mod._fp8_saved_scale_refs.clear()
|
||||
|
||||
save_file = tmp_path / "auto_scales.json"
|
||||
with _build_attention(
|
||||
MultiModalConfig(
|
||||
mm_encoder_attn_dtype="fp8",
|
||||
mm_encoder_fp8_scale_save_path=str(save_file),
|
||||
)
|
||||
) as attn:
|
||||
if attn is None or not attn.fp8_enabled:
|
||||
pytest.skip("FP8 attention not available")
|
||||
|
||||
attn = attn.to("cuda")
|
||||
S, H, D = 16, NUM_HEADS, HEAD_DIM
|
||||
|
||||
# Run exactly _FP8_AMAX_HISTORY_LEN forward passes.
|
||||
for i in range(_FP8_AMAX_HISTORY_LEN):
|
||||
mag = float(i + 1)
|
||||
q = torch.full((S, H, D), mag, device="cuda", dtype=torch.bfloat16)
|
||||
k = torch.full((S, H, D), mag * 0.5, device="cuda", dtype=torch.bfloat16)
|
||||
v = torch.full((S, H, D), mag * 0.3, device="cuda", dtype=torch.bfloat16)
|
||||
attn._record_amax_and_update_scales(q, k, v)
|
||||
|
||||
# File should have been written on the 16th call (buffer wrap).
|
||||
assert save_file.is_file(), "Scale file was not saved"
|
||||
scales = json.loads(save_file.read_text())
|
||||
assert LAYER_0 in scales
|
||||
assert set(scales[LAYER_0].keys()) == {"q", "k", "v"}
|
||||
for val in scales[LAYER_0].values():
|
||||
assert isinstance(val, float) and val > 0
|
||||
|
||||
# Path is cleared after the one-shot save fires.
|
||||
assert _mod._fp8_scale_save_path is None
|
||||
@@ -1,168 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the fused vocab-parallel embedding kernel
|
||||
(``vllm._custom_ops.vocab_parallel_embedding``).
|
||||
|
||||
The guarantee: the kernel is a drop-in for the TP > 1 embedding path, i.e. it
|
||||
returns bit-exactly what mask + shift + gather + ``masked_fill_`` returns on
|
||||
each rank, and summing the per-rank outputs (what the all-reduce does)
|
||||
reconstructs a full-table lookup.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
get_masked_input_and_mask,
|
||||
pad_vocab_size,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
requires_cuda = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="the fused vocab-parallel embedding kernel is CUDA-only",
|
||||
)
|
||||
|
||||
|
||||
def _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size, pad=64):
|
||||
return VocabParallelEmbedding._get_indices(
|
||||
pad_vocab_size(vocab_size, pad),
|
||||
pad_vocab_size(org_vocab_size, pad),
|
||||
vocab_size,
|
||||
org_vocab_size,
|
||||
tp_rank,
|
||||
tp_size,
|
||||
)
|
||||
|
||||
|
||||
def _reference(input_ids, weight, si):
|
||||
"""The path the kernel replaces."""
|
||||
masked_input, input_mask = get_masked_input_and_mask(
|
||||
input_ids,
|
||||
si.org_vocab_start_index,
|
||||
si.org_vocab_end_index,
|
||||
si.num_org_vocab_padding,
|
||||
si.added_vocab_start_index,
|
||||
si.added_vocab_end_index,
|
||||
)
|
||||
out = torch.nn.functional.embedding(masked_input.long(), weight)
|
||||
return out.masked_fill_(input_mask.unsqueeze(-1), 0)
|
||||
|
||||
|
||||
def _fused(input_ids, weight, si):
|
||||
return ops.vocab_parallel_embedding(
|
||||
input_ids,
|
||||
weight,
|
||||
si.org_vocab_start_index,
|
||||
si.org_vocab_end_index,
|
||||
si.num_org_vocab_padding,
|
||||
si.added_vocab_start_index,
|
||||
si.added_vocab_end_index,
|
||||
)
|
||||
|
||||
|
||||
def _shard(full_table, si, rows_per_rank, dtype):
|
||||
"""Lay a rank's rows out the way the weight loader does."""
|
||||
weight = torch.zeros(rows_per_rank, full_table.shape[1], dtype=dtype, device="cuda")
|
||||
num_org = si.org_vocab_end_index - si.org_vocab_start_index
|
||||
weight[:num_org] = full_table[si.org_vocab_start_index : si.org_vocab_end_index]
|
||||
added_at = num_org + si.num_org_vocab_padding
|
||||
num_added = si.added_vocab_end_index - si.added_vocab_start_index
|
||||
weight[added_at : added_at + num_added] = full_table[
|
||||
si.added_vocab_start_index : si.added_vocab_end_index
|
||||
]
|
||||
return weight
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("tp_size", [2, 4, 8])
|
||||
@pytest.mark.parametrize(
|
||||
"vocab_size,org_vocab_size", [(128256, 128256), (32256, 32000)]
|
||||
)
|
||||
@pytest.mark.parametrize("hidden", [4096, 3584])
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
|
||||
"""Every rank's partial output is bit-exact with the eager path, including
|
||||
the added-vocab (LoRA) rows and the ids owned by another rank."""
|
||||
set_random_seed(7)
|
||||
dtype = torch.bfloat16
|
||||
rows = pad_vocab_size(vocab_size, 64) // tp_size
|
||||
full_table = torch.randn(pad_vocab_size(vocab_size, 64), hidden, dtype=dtype).cuda()
|
||||
ids = torch.randint(0, vocab_size, (257,), dtype=id_dtype, device="cuda")
|
||||
|
||||
for tp_rank in range(tp_size):
|
||||
si = _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size)
|
||||
weight = _shard(full_table, si, rows, dtype)
|
||||
torch.testing.assert_close(
|
||||
_fused(ids, weight, si), _reference(ids, weight, si), atol=0.0, rtol=0.0
|
||||
)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("tp_size", [2, 8])
|
||||
def test_rank_sum_is_full_lookup(tp_size):
|
||||
"""What the all-reduce sees: the partials add up to a full-table lookup."""
|
||||
set_random_seed(11)
|
||||
vocab_size, org_vocab_size, hidden = 32256, 32000, 2048
|
||||
dtype = torch.bfloat16
|
||||
rows = pad_vocab_size(vocab_size, 64) // tp_size
|
||||
full_table = torch.randn(pad_vocab_size(vocab_size, 64), hidden, dtype=dtype).cuda()
|
||||
# Ids from both the base and the added (LoRA) range.
|
||||
ids = torch.randint(0, vocab_size, (129,), dtype=torch.int32, device="cuda")
|
||||
ids[:16] = torch.randint(
|
||||
org_vocab_size, vocab_size, (16,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
total = torch.zeros(ids.shape[0], hidden, dtype=dtype, device="cuda")
|
||||
for tp_rank in range(tp_size):
|
||||
si = _shard_indices(vocab_size, org_vocab_size, tp_rank, tp_size)
|
||||
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
|
||||
|
||||
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
|
||||
"""SP's deferred reduction preserves masked contributions and their dtype."""
|
||||
with torch.device("cuda"):
|
||||
values = torch.arange(128 * 16).reshape(128, 16)
|
||||
table = ((values % 29 - 14) / 16).to(dtype)
|
||||
ids = torch.tensor([0, 63, 64, 127, 12])
|
||||
expected = table[ids].float()
|
||||
for rank in range(2):
|
||||
local = expected.clone()
|
||||
local[ids // 64 != rank] = 0
|
||||
remote = (expected - local).to(dtype)
|
||||
group = Mock(world_size=2, rank_in_group=rank)
|
||||
# Model the other vocabulary owner's contribution, including FP8 bytes.
|
||||
group.all_reduce.side_effect = lambda output, remote=remote: (
|
||||
output + remote.view(output.dtype)
|
||||
)
|
||||
for reduce_results in (True, False):
|
||||
embedding = VocabParallelEmbedding(
|
||||
128,
|
||||
16,
|
||||
params_dtype=dtype,
|
||||
parallel_group=group,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
embedding.use_fused_embedding = use_fused
|
||||
embedding.weight_loader(embedding.get_parameter("weight"), table)
|
||||
actual = embedding(ids)
|
||||
assert actual.dtype == dtype
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected if reduce_results else local,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
Reference in New Issue
Block a user