[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,407 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import csv
import importlib
import importlib.util
import os
import pytest
import torch
from tests.utils import TestFP8Layer
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = [
pytest.mark.skipif(
not (
current_platform.is_rocm()
and current_platform.supports_fp8()
and aiter_available
),
reason="Requires ROCm + FP8 support + aiter",
),
pytest.mark.usefixtures("default_vllm_config"),
]
@pytest.fixture
def enable_hipb_mm_kernel(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
rocm_aiter_ops.refresh_env_variables()
yield
rocm_aiter_ops.refresh_env_variables()
def _make_config(
*,
weight_quant_key=kFp8StaticChannelSym,
out_dtype: torch.dtype = torch.bfloat16,
weight_shape: tuple[int, int] = (512, 4096),
) -> FP8ScaledMMLinearLayerConfig:
return FP8ScaledMMLinearLayerConfig(
weight_quant_key=weight_quant_key,
activation_quant_key=kFp8DynamicTokenSym,
weight_shape=weight_shape,
input_dtype=torch.bfloat16,
out_dtype=out_dtype,
)
def _find_csv_row(path: str, m: int, n: int, k: int) -> dict | None:
if not os.path.exists(path):
return None
with open(path, newline="") as f:
reader = csv.DictReader(f, skipinitialspace=True)
for row in reader:
try:
if (
int(row.get("m", -1)) == m
and int(row.get("n", -1)) == n
and int(row.get("k", -1)) == k
):
return dict(row)
except (TypeError, ValueError):
continue
return None
def _skip_if_no_hipb_mm_solution(exc: RuntimeError) -> None:
if "hipblasLtMatmulAlgoGetHeuristic found 0 valid solutions" in str(exc):
pytest.skip(
"hipb_mm bpreshuffle path has no valid hipBLASLt solution on "
"this ROCm stack."
)
def _check_bpreshuffle_runtime_support(weight_shape: tuple[int, int], num_tokens: int):
import aiter
from aiter.ops.shuffle import shuffle_weight
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device="cuda")
w = torch.randn(weight_shape, dtype=torch.bfloat16, device="cuda")
aiter.hipb_create_extension()
x_q, x_scale = aiter.pertoken_quant(x, quant_dtype=current_platform.fp8_dtype())
w_q, w_scale = aiter.pertoken_quant(w, quant_dtype=current_platform.fp8_dtype())
try:
aiter.hipb_mm(
x_q,
shuffle_weight(w_q, layout=(16, 16)).t(),
solution_index=-1,
out_dtype=torch.bfloat16,
scaleA=x_scale,
scaleB=w_scale.t().contiguous(),
scaleOut=None,
bpreshuffle=True,
)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
def test_hipb_mm_kernel_requires_hipbmm_flag(monkeypatch: pytest.MonkeyPatch):
# The kernel rejects when `is_hip_fp8bmm_enabled()` is False. That helper
# requires AITER + AITER_LINEAR + MI3xx, so dropping AITER_LINEAR exercises
# the rejection branch.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "0")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", raising=False)
rocm_aiter_ops.refresh_env_variables()
is_supported, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.is_supported()
assert not is_supported
assert reason == (
"requires setting `VLLM_ROCM_USE_AITER=1`, "
"`VLLM_ROCM_USE_AITER_LINEAR=1`, "
"and `VLLM_ROCM_USE_AITER_LINEAR_HIPBMM=1`."
)
def test_hipb_mm_flag_enables_hip_online_tuning(
monkeypatch: pytest.MonkeyPatch,
):
import vllm.envs as envs_mod
import vllm.platforms.rocm as rocm_mod
# The rocm.py gate requires all three AITER flags (and MI3xx) to auto-set
# HIP_ONLINE_TUNING.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
try:
importlib.reload(envs_mod)
importlib.reload(rocm_mod)
assert envs_mod.VLLM_ROCM_USE_AITER
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
assert os.environ.get("HIP_ONLINE_TUNING") == "1"
finally:
monkeypatch.undo()
os.environ.pop("HIP_ONLINE_TUNING", None)
importlib.reload(envs_mod)
importlib.reload(rocm_mod)
rocm_aiter_ops.refresh_env_variables()
def test_hipb_mm_kernel_can_implement_success(enable_hipb_mm_kernel):
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
_make_config()
)
assert can_implement
assert reason is None
@pytest.mark.parametrize(
("config", "expected_reason"),
[
(
_make_config(weight_quant_key=kFp8StaticTensorSym),
"requires per token activation scales and per channel weight scales.",
),
(
_make_config(out_dtype=torch.float16),
"requires bfloat16 output dtype.",
),
(
_make_config(weight_shape=(8, 4090)),
"requires N >= 16 and both N and K divisible by 16, "
"received N=8 and K=4090.",
),
],
)
def test_hipb_mm_kernel_can_implement_rejects_unsupported_configs(
enable_hipb_mm_kernel,
config: FP8ScaledMMLinearLayerConfig,
expected_reason: str,
):
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
config
)
assert not can_implement
assert reason == expected_reason
def test_hipb_mm_kernel_process_weights_after_loading_shuffles_weights(
enable_hipb_mm_kernel,
):
weight_shape = (512, 4096)
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
_make_config(weight_shape=weight_shape),
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.rand(weight_shape, device="cuda").to(current_platform.fp8_dtype()).t(),
requires_grad=False,
)
layer.weight_scale = torch.nn.Parameter(
torch.rand((weight_shape[0], 1), dtype=torch.float32, device="cuda"),
requires_grad=False,
)
layer.input_scale = None
layer.input_scale_ub = None
original_weight = layer.weight.detach().clone()
original_weight_scale = layer.weight_scale.detach().clone()
kernel.process_weights_after_loading(layer)
# process_weights_after_loading now pre-applies the transposes that used
# to live in _rocm_aiter_hipb_mm_fp8_impl, so the stored weight is the
# shuffled tensor with a trailing `.t()` view, and the stored weight scale
# is its transposed-contiguous form.
expected_weight = rocm_aiter_ops.shuffle_weight(
original_weight.t().contiguous()
).t()
torch.testing.assert_close(layer.weight, expected_weight)
expected_weight_scale = original_weight_scale.t().contiguous()
torch.testing.assert_close(layer.weight_scale, expected_weight_scale)
def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel):
import aiter
weight_shape = (512, 4096)
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=32)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticChannelSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
# hipb_mm uses a transposed-result GEMM internally, so the flattened token
# count becomes the effective N dimension passed into hipBLASLt. Keep it
# aligned to avoid heuristic failures for tiny N.
x = torch.randn(2, 16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device="cuda")
try:
out = layer(x, bias)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
x_2d = x.view(-1, x.shape[-1])
x_q, x_scale = layer.kernel.quant_fp8(
x_2d,
layer.input_scale,
layer.input_scale_ub,
)
try:
# process_weights_after_loading already applies the trailing `.t()` on
# the shuffled weight and the `.t().contiguous()` on the weight scale,
# so the raw aiter call uses them directly.
expected = aiter.hipb_mm(
x_q,
layer.weight,
solution_index=-1,
bias=bias,
out_dtype=torch.bfloat16,
scaleA=x_scale,
scaleB=layer.weight_scale,
scaleOut=None,
bpreshuffle=True,
).view(*out.shape)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
assert isinstance(layer.kernel, AiterHipbMMPerTokenFp8ScaledMMLinearKernel)
assert out.shape == (2, 16, weight_shape[0])
torch.testing.assert_close(out, expected)
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
"""Kernel output should match a dequantized fp32 reference within
fp8 per-token / per-channel quantization noise."""
weight_shape = (512, 4096) # (N, K)
num_tokens = 32
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
fp8_dtype = current_platform.fp8_dtype()
fp8_max = get_fp8_min_max()[1]
device = torch.device("cuda")
# Build a bf16 weight and quantize per output channel (one scale per row).
w_bf16 = torch.randn(weight_shape, dtype=torch.bfloat16, device=device)
w_amax = w_bf16.abs().amax(dim=1, keepdim=True).to(torch.float32)
w_scale = (w_amax / fp8_max).clamp(min=1e-12)
w_fp8 = (w_bf16.to(torch.float32) / w_scale).clamp(-fp8_max, fp8_max).to(fp8_dtype)
w_dequant = w_fp8.to(torch.float32) * w_scale
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device=device)
layer = torch.nn.Module()
# Pre-`process_weights_after_loading` convention: weight stored as the
# `[K, N]` view of the fp8 tensor.
layer.weight = torch.nn.Parameter(w_fp8.t(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale, requires_grad=False)
layer.input_scale = None
layer.input_scale_ub = None
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
_make_config(weight_shape=weight_shape),
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
)
kernel.process_weights_after_loading(layer)
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device=device)
try:
out = kernel.apply_weights(layer, x, bias)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
# Reference: quantize x per-token the same way the kernel does, then run
# the matmul in fp32 against the dequantized weight. This isolates plumbing
# / reduction bugs from inherent fp8 quantization noise.
x_amax = x.abs().amax(dim=1, keepdim=True).to(torch.float32)
x_scale_ref = (x_amax / fp8_max).clamp(min=1e-12)
x_q = (x.to(torch.float32) / x_scale_ref).clamp(-fp8_max, fp8_max).to(fp8_dtype)
x_dequant = x_q.to(torch.float32) * x_scale_ref
expected = (x_dequant @ w_dequant.t() + bias.to(torch.float32)).to(torch.bfloat16)
assert out.shape == (num_tokens, weight_shape[0])
# K=4096 fp8 reduction leaves room for accumulation order drift and
# catastrophic cancellation on near-zero outputs; tolerances are loose
# enough to absorb that but tight enough to catch wrong layouts, missing
# bias, swapped scales, etc.
torch.testing.assert_close(out, expected, atol=5.0, rtol=0.1)
def test_hipb_mm_kernel_online_tuning_writes_csv(
enable_hipb_mm_kernel,
monkeypatch: pytest.MonkeyPatch,
tmp_path,
):
weight_shape = (256, 4096)
cache_file = tmp_path / "hip_online_tuning_res.csv"
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=16)
monkeypatch.setenv("HIP_ONLINE_TUNING", "1")
monkeypatch.chdir(tmp_path)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticChannelSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
# The effective heuristic N dimension is the flattened token count.
x = torch.randn(16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
try:
out = layer(x)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
torch.accelerator.synchronize()
assert out.shape == (16, weight_shape[0])
assert cache_file.exists()
# hipb_mm records the internal GEMM dimensions used by hipBLASLt after its
# transposed-result transformation.
row = _find_csv_row(
str(cache_file),
m=weight_shape[0],
n=x.shape[0],
k=weight_shape[1],
)
assert row is not None
-29
View File
@@ -1,29 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.utils import opcheck
from vllm import _custom_ops as ops # noqa: F401
@pytest.mark.skipif(
not hasattr(torch.ops._C, "awq_dequantize"),
reason="AWQ is not supported on this GPU type.",
)
def test_awq_dequantize_opcheck(monkeypatch: pytest.MonkeyPatch):
with monkeypatch.context() as m:
m.setenv("VLLM_USE_TRITON_AWQ", "0")
qweight = torch.randint(
-2000000000, 2000000000, (8192, 256), device="cuda", dtype=torch.int32
)
scales = torch.rand((64, 2048), device="cuda", dtype=torch.float16)
zeros = torch.empty((64, 256), device="cuda", dtype=torch.int32)
split_k_iters = 0
thx = 0
thy = 0
opcheck(
torch.ops._C.awq_dequantize,
(qweight, scales, zeros, split_k_iters, thx, thy),
)
@@ -1,177 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the AWQ Triton kernel.
Run `pytest tests/kernels/quantization/test_awq_triton.py`.
"""
import pytest
import torch
from vllm.model_executor.layers.quantization.awq_triton import (
AWQ_TRITON_SUPPORTED_GROUP_SIZES,
awq_dequantize_triton,
awq_gemm_triton,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="AWQ Triton kernels require CUDA/ROCm or XPU.",
)
device = current_platform.device_type
def reverse_awq_order(t: torch.Tensor):
bits = 4
AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
reverse_order_tensor = torch.arange(
t.shape[-1],
dtype=torch.int32,
device=t.device,
)
reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
reverse_order_tensor = reverse_order_tensor.view(-1)
t = t[:, reverse_order_tensor] & 0xF
return t
# qweights - [R , C // 8], int32
# scales - [R // G, C ], float16
# zeros - [R // G, C // 8], int32
def awq_dequantize_torch(
qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, group_size: int
) -> torch.Tensor:
if group_size == -1:
group_size = qweight.shape[0]
bits = 4
shifts = torch.arange(0, 32, bits, device=qzeros.device)
iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
torch.int8
)
iweights = iweights.view(iweights.shape[0], -1)
zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
torch.int8
)
zeros = zeros.view(qzeros.shape[0], -1)
zeros = reverse_awq_order(zeros)
iweights = reverse_awq_order(iweights)
iweights = torch.bitwise_and(iweights, (2**bits) - 1)
zeros = torch.bitwise_and(zeros, (2**bits) - 1)
scales = scales.repeat_interleave(group_size, dim=0)
zeros = zeros.repeat_interleave(group_size, dim=0)
return (iweights - zeros) * scales
# qweights - [R , C // 8], int32
# scales - [R // G, C ], float16
# zeros - [R // G, C // 8], int32
@pytest.mark.parametrize("qweight_rows", [3584, 18944, 128, 256, 512, 1024])
@pytest.mark.parametrize("qweight_cols", [448, 576, 4736, 16, 32, 64, 128])
@pytest.mark.parametrize("group_size", AWQ_TRITON_SUPPORTED_GROUP_SIZES)
def test_dequantize(qweight_rows, qweight_cols, group_size):
if group_size == -1:
group_size = qweight_rows
qweight_dtype = torch.int32
scales_rows = qweight_rows // group_size
scales_cols = qweight_cols * 8
scales_dtype = torch.float16
zeros_rows = scales_rows
zeros_cols = qweight_cols
zeros_dtype = torch.int32
set_random_seed(0)
qweight = torch.randint(
0,
torch.iinfo(torch.int32).max,
(qweight_rows, qweight_cols),
dtype=qweight_dtype,
device=device,
)
scales = torch.rand(scales_rows, scales_cols, dtype=scales_dtype, device=device)
zeros = torch.randint(
0,
torch.iinfo(torch.int32).max,
(zeros_rows, zeros_cols),
dtype=zeros_dtype,
device=device,
)
iweights_triton = awq_dequantize_triton(qweight, scales, zeros)
assert not torch.any(torch.isinf(iweights_triton)) and not torch.any(
torch.isnan(iweights_triton)
)
iweights_torch = awq_dequantize_torch(qweight, scales, zeros, group_size)
torch.testing.assert_close(iweights_triton, iweights_torch)
# input - [N, K]
# qweight - [K, M // 8]
# qzeros - [K // G, M // 8]
# scales - [K // G, M]
@pytest.mark.parametrize("N", [1, 2, 4, 8, 14, 17, 23, 32])
@pytest.mark.parametrize("K", [128])
@pytest.mark.parametrize("M", [16, 24, 32])
@pytest.mark.parametrize("group_size", AWQ_TRITON_SUPPORTED_GROUP_SIZES)
@pytest.mark.parametrize("splitK", [1, 8])
def test_gemm(N, K, M, splitK, group_size):
if group_size == -1:
group_size = K
split_k_iters = splitK
input_rows = N
input_cols = K
input_dtype = torch.float32
qweight_rows = input_cols
qweight_cols = M // 8
scales_rows = qweight_rows // group_size
scales_cols = M
scales_dtype = torch.float32
qzeros_rows = scales_rows
qzeros_cols = qweight_cols
set_random_seed(0)
input = torch.rand((input_rows, input_cols), dtype=input_dtype, device=device)
qweight = torch.randint(
0, torch.iinfo(torch.int32).max, (qweight_rows, qweight_cols), device=device
)
qzeros = torch.randint(
0, torch.iinfo(torch.int32).max, (qzeros_rows, qzeros_cols), device=device
)
scales = torch.rand((scales_rows, scales_cols), dtype=scales_dtype, device=device)
output_triton = awq_gemm_triton(input, qweight, scales, qzeros, split_k_iters)
assert not torch.any(torch.isinf(output_triton)) and not torch.any(
torch.isnan(output_triton)
)
dequantized_weights = awq_dequantize_triton(qweight, scales, qzeros)
output_torch = torch.matmul(input, dequantized_weights)
assert not torch.any(torch.isinf(output_torch)) and not torch.any(
torch.isnan(output_torch)
)
torch.testing.assert_close(
output_triton.cpu(), output_torch.cpu(), atol=1e-1, rtol=1e-1
)
@@ -1,423 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/sgl-project/sglang/pull/2575
import itertools
import types
import pytest
import torch
from tests.kernels.quant_utils import (
native_per_token_group_quant_fp8,
native_w8a8_block_matmul,
)
from tests.kernels.utils import fp8_ulp_distance
from vllm.config import VllmConfig
from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
B12xFp8BlockScaledMMKernel,
_run_b12x_fp8_block_scaled_mm,
)
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import cutlass_scaled_mm
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
w8a8_triton_block_scaled_mm,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
fp8_gemm_nt,
get_tma_aligned_size,
is_deep_gemm_e8m0_used,
per_block_cast_to_fp8,
should_use_deepgemm_for_fp8_linear,
)
from vllm.utils.flashinfer import (
flashinfer_fp8_blockscale_gemm,
has_flashinfer_fp8_blockscale_gemm,
)
from vllm.utils.import_utils import has_deep_gemm
capability = current_platform.get_device_capability()
if capability is None or capability < (9, 0):
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
vllm_config = VllmConfig()
# Test configurations
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
# Quantization test configs
NUM_TOKENS = [7, 2050]
D = [512, 4096, 5120, 13824]
GROUP_SIZE = [64, 128, 512]
COLUMN_MAJOR_SCALES = [True, False]
TMA_ALIGNED_SCALES = [True, False]
# Matmul test configs
M = [1, 7, 8, 83, 4096]
N = [128, 512, 576, 7168, 13824]
K = [256, 3884, 4096, 13824, 16384]
# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
# and its hidden size is 7168.
BLOCK_SIZE = [[128, 128]]
OUT_DTYPES = [torch.bfloat16] # [torch.float32, torch.half, torch.bfloat16]
SEEDS = [0]
# Skip all tests if CUDA is not available
pytest.importorskip("torch.cuda")
@pytest.fixture(autouse=True)
def setup_cuda():
torch.set_default_device("cuda")
@pytest.mark.skipif(
current_platform.is_fp8_fnuz(),
reason="This platform supports e4m3fnuz, not e4m3fn.",
)
@pytest.mark.parametrize(
"num_tokens,d,dtype,group_size,column_major_scales,tma_aligned_scales,seed",
itertools.product(
NUM_TOKENS,
D,
DTYPES,
GROUP_SIZE,
COLUMN_MAJOR_SCALES,
TMA_ALIGNED_SCALES,
SEEDS,
),
)
@torch.inference_mode()
def test_per_token_group_quant_fp8(
num_tokens, d, dtype, group_size, column_major_scales, tma_aligned_scales, seed
):
torch.manual_seed(seed)
x = torch.rand(num_tokens, d, dtype=dtype)
ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size)
out, scale = per_token_group_quant_fp8(
x,
group_size,
column_major_scales=column_major_scales,
tma_aligned_scales=tma_aligned_scales,
)
if current_platform.is_rocm():
# On gfx950 the Triton and PyTorch FP8 kernels can round in opposite
# directions when an element lands at the midpoint between two adjacent
# e4m3fn values (1-ULP tie-breaking). Verify: (1) no element is more
# than 1 FP8 ULP away, and (2) fewer than 0.05% of elements have any
# mismatch. Observed worst case across all parameter combos: 0.049%,
# max ULP = 1.
ulp = fp8_ulp_distance(out, ref_out)
assert (ulp <= 1).all(), (
f"FP8 mismatch > 1 ULP: {int((ulp > 1).sum())} elements"
)
assert float((ulp > 0).float().mean()) < 5e-4, (
f"Too many 1-ULP mismatches: {int((ulp > 0).sum())}/{ulp.numel()}"
)
else:
assert torch.allclose(
out.to(torch.float32), ref_out.to(torch.float32), rtol=0.15
)
assert torch.allclose(scale, ref_scale)
if column_major_scales:
assert scale.stride()[-2] == 1
if tma_aligned_scales:
assert scale.stride()[-1] == get_tma_aligned_size(num_tokens, 4)
@pytest.mark.parametrize(
"M,N,K,block_size,out_dtype,seed",
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
)
@torch.inference_mode()
def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
torch.manual_seed(seed)
factor_for_scale = 1e-2
fp8_info = torch.finfo(current_platform.fp8_dtype())
fp8_max, fp8_min = fp8_info.max, fp8_info.min
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(current_platform.fp8_dtype())
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(current_platform.fp8_dtype())
block_n, block_k = block_size[0], block_size[1]
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
out = w8a8_triton_block_scaled_mm(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.001
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform."
)
@pytest.mark.parametrize(
# 65/66/67 cover all M%4 residue classes above the SM100 swapAB
# threshold (m <= 64); 1026 crosses multiple 128-row SF atoms.
"M",
[32, 65, 66, 67, 1026],
)
@torch.inference_mode()
def test_w8a8_block_fp8_cutlass_matmul(M):
# Test simple case where weight.shape % 128 != 0,
# like in DSV3 kv_a_proj_with_mqa
N = 576
K = 7168
block_size = [128, 128]
out_dtype = torch.bfloat16
seed = 0
torch.manual_seed(seed)
factor_for_scale = 1e-2
fp8_info = torch.finfo(torch.float8_e4m3fn)
fp8_max, fp8_min = fp8_info.max, fp8_info.min
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
block_n, block_k = block_size[0], block_size[1]
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
A_fp8, As = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=False
)
# CUTLASS uses column-major format for scales
A_fp8_cutlass, As_cutlass = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True
)
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
out = cutlass_scaled_mm(A_fp8_cutlass, B_fp8, As_cutlass, Bs, block_size, out_dtype)
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.001
@pytest.mark.skipif(
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
reason="torch._scaled_mm DeepSeek-style block scaling only supports SM90.",
)
def test_w8a8_block_fp8_torch_scaled_mm_matmul():
# BlockWiseTorchFP8ScaledMMLinearKernel: 1x128 activation + 128x128 weight
# block scaling routed through torch._scaled_mm. M=83 is not a multiple of
# 4 so this also exercises the M-padding path.
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
BlockWiseTorchFP8ScaledMMLinearKernel,
)
M = 83
N = 576
K = 7168
block_size = [128, 128]
out_dtype = torch.bfloat16
seed = 0
torch.manual_seed(seed)
factor_for_scale = 1e-2
fp8_info = torch.finfo(torch.float8_e4m3fn)
fp8_max, fp8_min = fp8_info.max, fp8_info.min
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
block_n, block_k = block_size[0], block_size[1]
n_tiles = (N + block_n - 1) // block_n
k_tiles = (K + block_k - 1) // block_k
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
# Reference uses row-major activation scales.
A_fp8, As = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=False
)
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
# The kernel expects column-major activation scales on CUDA.
A_fp8_cuda, As_cuda = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True
)
stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__(
BlockWiseTorchFP8ScaledMMLinearKernel
)
stub.config = types.SimpleNamespace(out_dtype=out_dtype)
out = stub.apply_block_scaled_mm(
A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda()
)
ref_out = ref_out.cuda()
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.001
@pytest.mark.skipif(
current_platform.is_fp8_fnuz(),
reason="This platform supports e4m3fnuz, not e4m3fn.",
)
@pytest.mark.parametrize(
"M,N,K,block_size,out_dtype,seed",
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
)
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGemm kernels not available.")
@torch.inference_mode()
def test_w8a8_block_fp8_deep_gemm_matmul(M, N, K, block_size, out_dtype, seed):
torch.manual_seed(seed)
fp8_info = torch.finfo(torch.float8_e4m3fn)
fp8_max = fp8_info.max
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
# only aligned sizes are supported by deepgemm
if not should_use_deepgemm_for_fp8_linear(
output_dtype=out_dtype, weight_shape=B_fp32.shape, supports_deep_gemm=True
):
pytest.skip(f"Skipping test; invalid size {M}, {N}, {K}")
A_fp8, As_fp8 = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True, tma_aligned_scales=True
)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(
B_fp32,
block_size=block_size,
use_ue8m0=is_deep_gemm_e8m0_used(),
)
As = As_fp8.to(torch.float32)
Bs = Bs_fp8.to(torch.float32)
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
out = torch.zeros((M, N), device="cuda", dtype=out_dtype)
assert As_fp8.shape == (M, (K + 127) // 128), (
f"{As_fp8.shape} != {(M, (K + 127) // 128)}"
)
fp8_gemm_nt((A_fp8, As_fp8), (B_fp8, Bs_fp8), out)
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.001
@pytest.mark.skipif(
current_platform.is_fp8_fnuz(),
reason="This platform supports e4m3fnuz, not e4m3fn.",
)
@pytest.mark.parametrize(
"M,N,K,block_size,out_dtype,seed",
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
)
@torch.inference_mode()
def test_w8a8_block_fp8_flashinfer_matmul(M, N, K, block_size, out_dtype, seed):
if not has_flashinfer_fp8_blockscale_gemm():
pytest.skip(
"FlashInfer block GEMM not available (requires SM90+ and FlashInfer)"
)
# only aligned sizes
if K % 128 != 0 or N % 64 != 0:
pytest.skip(f"Skipping test; invalid size {M}, {N}, {K}")
torch.manual_seed(seed)
fp8_info = torch.finfo(torch.float8_e4m3fn)
fp8_max = fp8_info.max
A_bf16 = (torch.rand(M, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
B_bf16 = (torch.rand(N, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
A_fp8, As_fp8 = per_token_group_quant_fp8(A_bf16, block_size[1], use_ue8m0=False)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(B_bf16, block_size, use_ue8m0=False)
As = As_fp8.to(torch.float32)
Bs = Bs_fp8.to(torch.float32)
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
out = flashinfer_fp8_blockscale_gemm(
input=A_bf16,
weight=B_fp8,
input_scale=None,
weight_scale=Bs,
out_dtype=out_dtype,
)
rel_diff = torch.mean(
torch.abs(out.to(torch.bfloat16) - ref_out.to(torch.bfloat16))
) / torch.mean(torch.abs(ref_out.to(torch.bfloat16)))
assert rel_diff < 0.001
@pytest.mark.parametrize(
"M,N,K",
[(1, 128, 256), (8, 256, 512), (129, 256, 256), (2, 4096, 4096)],
)
@torch.inference_mode()
def test_w8a8_block_fp8_b12x_matmul(M, N, K):
supported, reason = B12xFp8BlockScaledMMKernel.is_supported()
if not supported:
pytest.skip(reason)
torch.manual_seed(M)
fp8_max = torch.finfo(torch.float8_e4m3fn).max
A_bf16 = (torch.rand(M, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
B_bf16 = (torch.rand(N, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
A_fp8, As = per_token_group_quant_fp8(A_bf16, 128, use_ue8m0=False)
B_fp8, Bs = per_block_cast_to_fp8(
B_bf16,
block_size=[128, 128],
use_ue8m0=False,
)
As = As.float()
Bs = Bs.float()
ref_out = native_w8a8_block_matmul(
A_fp8,
B_fp8,
As,
Bs,
[128, 128],
torch.bfloat16,
)
out = _run_b12x_fp8_block_scaled_mm(
A_fp8,
B_fp8,
As,
Bs,
torch.bfloat16,
)
rel_diff = torch.mean(torch.abs(out.float() - ref_out.float())) / torch.mean(
torch.abs(ref_out.float())
)
cosine = torch.nn.functional.cosine_similarity(
out.float().flatten(),
ref_out.float().flatten(),
dim=0,
)
# Four-way split-K uses atomic BF16 reductions, so nondeterministic atomic
# ordering can flap an output between adjacent BF16 values one ULP apart.
assert rel_diff < 0.003
assert cosine >= 0.99999
@@ -1,497 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU FP8 scaled-mm GEMM kernels.
Covers:
* fp8_scaled_mm_cpu (W8A16)
* fp8_scaled_mm_with_quant (W8A8)
Run `pytest tests/kernels/quantization/test_cpu_fp8_scaled_mm.py -v`.
"""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
CPUFp8PerTensorScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
requires_cpu_fp8_w8a16 = pytest.mark.skipif(
not ops._supports_cpu_fp8_w8a16, reason="fp8_scaled_mm_cpu op not available"
)
requires_cpu_fp8_w8a8 = pytest.mark.skipif(
not ops.cpu_has_amx_fp8(),
reason="requires native AMX-FP8 hardware",
)
if not torch.cpu._is_amx_tile_supported():
pytest.skip("requires AMX tile support", allow_module_level=True)
FP8_MAX = torch.finfo(torch.float8_e4m3fn).max
BLOCK_SIZE = [128, 128]
GROUP_SIZE = 128
def cdiv(a: int, b: int) -> int:
return -(a // -b)
# ---------------------------------------------------------------------------
# W8A16 helpers
# ---------------------------------------------------------------------------
def quantize_weight_block_fp8(
weight: torch.Tensor,
block_size: list[int],
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with block scales.
Returns:
fp8_weight: [N, K] float8_e4m3fn
scales: [n_tiles, k_tiles] float32
"""
N, K = weight.shape
block_n, block_k = block_size
fp8_max = torch.finfo(torch.float8_e4m3fn).max
n_tiles = cdiv(N, block_n)
k_tiles = cdiv(K, block_k)
# Pad for even blocking
pad_N = (block_n - (N % block_n)) % block_n
pad_K = (block_k - (K % block_k)) % block_k
if pad_N > 0 or pad_K > 0:
weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N))
# Reshape into blocks
w_blocks = weight.view(n_tiles, block_n, k_tiles, block_k)
w_blocks = w_blocks.permute(0, 2, 1, 3).contiguous()
# Per-block scale
abs_max = w_blocks.abs().amax(dim=(-2, -1), keepdim=True)
scales = abs_max / fp8_max
scales = torch.where(scales == 0, torch.ones_like(scales), scales)
# Quantize
q_fp8 = (w_blocks / scales).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
# Reshape back
fp8_weight = (
q_fp8.permute(0, 2, 1, 3)
.contiguous()
.view(N + pad_N, K + pad_K)[:N, :K]
.contiguous()
)
scales = scales.view(n_tiles, k_tiles)
return fp8_weight, scales
def dequant_weight_block_fp8(
fp8_weight: torch.Tensor,
scales: torch.Tensor,
block_size: list[int],
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Dequantize FP8 weight back to float for reference computation."""
N, K = fp8_weight.shape
block_n, block_k = block_size
n_tiles, k_tiles = scales.shape
pad_N = (block_n - (N % block_n)) % block_n
pad_K = (block_k - (K % block_k)) % block_k
if pad_N > 0 or pad_K > 0:
fp8_padded = torch.nn.functional.pad(fp8_weight.float(), (0, pad_K, 0, pad_N))
else:
fp8_padded = fp8_weight.float()
w_blocks = fp8_padded.view(n_tiles, block_n, k_tiles, block_k)
w_blocks = w_blocks.permute(0, 2, 1, 3).contiguous()
dq = w_blocks * scales.view(n_tiles, k_tiles, 1, 1)
dq = dq.permute(0, 2, 1, 3).contiguous().view(N + pad_N, K + pad_K)
return dq[:N, :K].to(out_dtype)
def ref_fp8_block_scaled_mm(
x: torch.Tensor,
fp8_weight: torch.Tensor,
scales: torch.Tensor,
block_size: list[int],
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Reference: dequant FP8→float32, matmul in float32, cast to out_dtype."""
w_dq = dequant_weight_block_fp8(fp8_weight, scales, block_size, torch.float32)
out = torch.mm(x.float(), w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
# ---------------------------------------------------------------------------
# W8A16 test: fp8_scaled_mm_cpu
# ---------------------------------------------------------------------------
M_SIZES = [1, 4, 16, 64, 128]
# (N, K) — weight shape is [N, K], output has N columns.
NK_SIZES = [
(128, 256),
(256, 512),
(512, 1024),
(1024, 2048),
(5120, 5120),
(17408, 5120),
(5120, 17408),
]
@requires_cpu_fp8_w8a16
@pytest.mark.parametrize("M", M_SIZES)
@pytest.mark.parametrize("N,K", NK_SIZES)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_scaled_mm(M: int, N: int, K: int, use_bias: bool):
"""fp8_scaled_mm_cpu correctness against float reference."""
torch.manual_seed(42)
out_dtype = torch.bfloat16
block_size = BLOCK_SIZE
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
fp8_weight, scales = quantize_weight_block_fp8(w_f32, block_size)
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
ref_out = ref_fp8_block_scaled_mm(
x, fp8_weight, scales, block_size, bias, out_dtype
)
packed_weight = torch.ops._C.convert_weight_packed(fp8_weight)
kernel_out = ops.fp8_scaled_mm_cpu(
x,
packed_weight,
scales,
block_size,
bias,
out_dtype,
True,
)
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
def quantize_weight_per_tensor_fp8(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with a single per-tensor scale."""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
scale = weight.abs().amax() / fp8_max
scale = torch.where(scale == 0, torch.ones_like(scale), scale)
q = (weight / scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
return q, scale
def ref_fp8_per_tensor_scaled_mm(
x: torch.Tensor,
fp8_weight: torch.Tensor,
scale: torch.Tensor,
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
w_dq = fp8_weight.float() * scale
out = torch.mm(x.float(), w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
# ---------------------------------------------------------------------------
# W8A8 helpers
# ---------------------------------------------------------------------------
def quantize_weight_per_tensor(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with a single per-tensor scale, broadcast
to [N, 1] (the shape float8_linear_prepack_cpu expects)."""
N = weight.shape[0]
abs_max = weight.abs().max()
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
return q, scale.expand(N).contiguous().view(N, 1)
def quantize_weight_per_channel(
weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with per-channel (per-row) scales [N, 1]."""
abs_max = weight.abs().amax(dim=1, keepdim=True)
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
return q, scale.float()
def quantize_weight_per_group(
weight: torch.Tensor,
group_size: int = GROUP_SIZE,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Quantize weight [N, K] to FP8 with per-group (128-block) scales [N, G]."""
N, K = weight.shape
G = K // group_size
w_groups = weight.view(N, G, group_size)
abs_max = w_groups.abs().amax(dim=2, keepdim=True)
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
q = (w_groups.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
return q.view(N, K), scale.squeeze(2).float()
def quantize_act_per_token(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Reference per-token (row) dynamic FP8 quantization of activations."""
x_f = x.float()
scale = (x_f.abs().amax(dim=1, keepdim=True) / FP8_MAX).clamp(min=1e-7)
scaled = (x_f / scale).clamp(-FP8_MAX, FP8_MAX)
# Round via fp16 to match the AVX10.2 kernel path.
q = scaled.half().to(torch.float8_e4m3fn)
return q, scale
def reference_static_fp8_linear(
x: torch.Tensor, # BF16 [M, K]
weight_fp8: torch.Tensor, # FP8 [N, K]
weight_scale: torch.Tensor, # float32 [N, 1]
act_scale: torch.Tensor, # float32 scalar
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Static per-tensor: quantize/dequant x with act_scale, dequant weight, matmul."""
x_scaled = (x.float() / act_scale.item()).clamp(-FP8_MAX, FP8_MAX)
# Round via fp16 to match the AVX10.2 kernel path.
x_q = x_scaled.half().to(torch.float8_e4m3fn)
x_dq = x_q.float() * act_scale.item()
w_dq = weight_fp8.float() * weight_scale.view(-1, 1)
out = torch.mm(x_dq, w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
NK_SIZES_PER_TENSOR = [
(32, 64),
(5120, 5120),
]
@pytest.mark.parametrize("M", [1, 64])
@pytest.mark.parametrize("N,K", NK_SIZES_PER_TENSOR)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_per_tensor_scaled_mm_kernel(
M: int, N: int, K: int, use_bias: bool, default_vllm_config
):
"""CPUFp8PerTensorScaledMMLinearKernel correctness against float reference.
Exercises the full kernel class (not just the raw op), including the
weight-orientation fixup: Fp8LinearMethod stores `layer.weight` as
[K, N] (torch._scaled_mm convention) before calling
process_weights_after_loading, so the kernel must transpose back to
[N, K] before VNNI-packing.
"""
torch.manual_seed(0)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
fp8_weight, scale = quantize_weight_per_tensor_fp8(w_f32)
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
ref_out = ref_fp8_per_tensor_scaled_mm(x, fp8_weight, scale, bias, out_dtype)
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(N, K),
input_dtype=out_dtype,
out_dtype=out_dtype,
)
kernel = CPUFp8PerTensorScaledMMLinearKernel(
config,
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
)
layer = torch.nn.Module()
# Fp8LinearMethod stores the weight transposed to [K, N] before calling
# process_weights_after_loading.
layer.register_parameter(
"weight", torch.nn.Parameter(fp8_weight.t().contiguous(), requires_grad=False)
)
layer.register_parameter(
"weight_scale", torch.nn.Parameter(scale.clone(), requires_grad=False)
)
kernel.process_weights_after_loading(layer)
kernel_out = kernel.apply_weights(layer, x, bias)
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
@pytest.mark.parametrize("n", [16, 48])
def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
"""The AMX tinygemm kernel tiles N in chunks of 32 and cannot handle a
remainder tile smaller than that, so can_implement must reject any N
that isn't a multiple of 32 rather than let it crash the process.
"""
config = FP8ScaledMMLinearLayerConfig(
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=kFp8StaticTensorSym,
weight_shape=(n, 64),
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
)
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
assert not supported
def reference_dynamic_fp8_linear(
x: torch.Tensor, # BF16 [M, K]
weight_fp8: torch.Tensor, # FP8 [N, K]
weight_scale: torch.Tensor, # float32 [N, G]
bias: torch.Tensor | None,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Dynamic per-token act x per-group weight: quantize x per-token,
dequant weight per-group, matmul."""
N, K = weight_fp8.shape
G = weight_scale.shape[1]
group_size = K // G
x_q, x_scale = quantize_act_per_token(x)
x_dq = x_q.float() * x_scale
w_dq = (
weight_fp8.float().view(N, G, group_size) * weight_scale.view(N, G, 1)
).view(N, K)
out = torch.mm(x_dq, w_dq.t())
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
# ---------------------------------------------------------------------------
# W8A8 tests: fp8_scaled_mm_with_quant
# ---------------------------------------------------------------------------
W8A8_M_SIZES = [1, 4, 16, 64]
W8A8_NK_SIZES = [(128, 256), (256, 512), (64, 128)]
@requires_cpu_fp8_w8a8
@pytest.mark.parametrize("static_activation", [True, False])
@pytest.mark.parametrize("M", W8A8_M_SIZES)
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_w8a8_per_tensor(
M: int, N: int, K: int, use_bias: bool, static_activation: bool
):
"""CPUFP8W8A8ScaledMMLinearKernel with per-tensor weight scale."""
torch.manual_seed(42)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=torch.bfloat16)
weight = torch.randn(N, K, dtype=torch.bfloat16)
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
w_fp8, w_scale = quantize_weight_per_tensor(weight)
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
if static_activation:
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
kernel_out = ops.fp8_scaled_mm_with_quant(
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
)
ref_out = reference_static_fp8_linear(
x, w_fp8, w_scale, act_scale, bias, out_dtype
)
atol = 0.05
else:
kernel_out = ops.fp8_scaled_mm_with_quant(
x, None, True, packed_weight, packed_scale, bias, out_dtype
)
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
atol = 0.05
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
@requires_cpu_fp8_w8a8
@pytest.mark.parametrize("static_activation", [True, False])
@pytest.mark.parametrize("M", W8A8_M_SIZES)
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_w8a8_per_channel(
M: int, N: int, K: int, use_bias: bool, static_activation: bool
):
"""CPUFP8W8A8ScaledMMLinearKernel with per-channel weight scale."""
torch.manual_seed(42)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=torch.bfloat16)
weight = torch.randn(N, K, dtype=torch.bfloat16)
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
w_fp8, w_scale = quantize_weight_per_channel(weight)
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
if static_activation:
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
kernel_out = ops.fp8_scaled_mm_with_quant(
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
)
ref_out = reference_static_fp8_linear(
x, w_fp8, w_scale, act_scale, bias, out_dtype
)
atol = 0.05
else:
kernel_out = ops.fp8_scaled_mm_with_quant(
x, None, True, packed_weight, packed_scale, bias, out_dtype
)
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
atol = 0.05
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
@requires_cpu_fp8_w8a8
@pytest.mark.parametrize("M", W8A8_M_SIZES)
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
@pytest.mark.parametrize("use_bias", [False, True])
def test_cpu_fp8_w8a8_per_group(M: int, N: int, K: int, use_bias: bool):
"""CPUFp8W8A8BlockScaledMMKernel: per-group (128-block) weight scale +
dynamic per-token activation scale."""
torch.manual_seed(42)
out_dtype = torch.bfloat16
x = torch.randn(M, K, dtype=torch.bfloat16)
weight = torch.randn(N, K, dtype=torch.bfloat16)
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
w_fp8, w_scale = quantize_weight_per_group(weight)
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
kernel_out = ops.fp8_scaled_mm_with_quant(
x, None, True, packed_weight, packed_scale, bias, out_dtype
)
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
assert kernel_out.dtype == out_dtype
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=0.05)
@@ -1,771 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cutlass kernels.
Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`.
"""
import random
import pytest
import torch
from tests.kernels.utils import baseline_scaled_mm, opcheck, to_fp8, to_int8
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
if not current_platform.is_cuda():
pytest.skip("These tests use CUTLASS which requires CUDA", allow_module_level=True)
MNK_FACTORS = [
(1, 256, 128),
(1, 16384, 1024),
(1, 24576, 496),
(16, 256, 496),
(16, 16384, 128),
(16, 24576, 4096),
(32, 8192, 4096),
(32, 16384, 4096),
(33, 1024, 1024),
(33, 8192, 128),
(64, 2048, 496),
(64, 16384, 1024),
(100, 8192, 496),
(128, 32768, 4096),
(256, 4096, 4096),
(512, 256, 1024),
(512, 8192, 4096),
(512, 16384, 128),
(512, 24576, 128),
]
# Shapes with N or K not divisible by 16. These exercise the padding path
# inside CutlassFP8ScaledMMLinearKernel.apply_scaled_mm (e.g. Qwen2.5-VL
# vision MLP dims).
UNALIGNED_MNK_FACTORS = [
(32, 3420, 1280),
(32, 1280, 6840),
(1, 3420, 1280),
(64, 6840, 1280),
(16, 100, 200),
(33, 255, 513),
]
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
# -1 means full extent in that dimension
TENSORWISE_GROUP_SHAPE = (-1, -1)
PER_TOKEN_GROUP_SHAPE = (1, -1)
PER_OUT_CH_GROUP_SHAPE = (-1, 1)
capability = current_platform.get_device_capability()
# Fall back to 0 so import doesn't fail on unknown capability
capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0
def rand_int8(shape: tuple, device: str = "cuda"):
return to_int8(torch.rand(shape, device=device) * 255 - 128)
def group_scale_helper(shape, group_shape):
return [shape[i] if s < 0 else s for i, s in enumerate(group_shape)]
def scale_shape(shape, group_shape):
assert len(shape) == len(group_shape)
group_shape = group_scale_helper(shape, group_shape)
return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape)))
def cutlass_fp8_gemm_helper(
m: int,
n: int,
k: int,
a_scale_group_shape: tuple,
b_scale_group_shape: tuple,
use_bias: bool,
out_dtype: type[torch.dtype] = torch.bfloat16,
device: str = "cuda",
):
# Test for a cutlass kernel with per-token activation quantization
# and per-output channel weight quantization.
a = to_fp8(torch.randn((m, k), device=device))
b = to_fp8(torch.randn((n, k), device=device).t())
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32)
scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32)
# make scales M-major for blockwise quant, doesn't affect 1D scales
scale_a = scale_a.t().contiguous().t()
# make scales K-major for blockwise quant, doesn't affect 1D scales
scale_b = scale_b.t().contiguous().t()
bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1)
opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias))
def cutlass_int8_gemm_helper(
m: int,
n: int,
k: int,
a_scale_group_shape: tuple,
b_scale_group_shape: tuple,
use_bias: bool,
out_dtype: type[torch.dtype] = torch.bfloat16,
device: str = "cuda",
):
# Test for a cutlass kernel with per-token activation quantization
# and per-output channel weight quantization.
a = to_int8(torch.randn((m, k), device=device) * 5)
b = to_int8(torch.randn((n, k), device=device).t() * 5)
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32)
scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32)
bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias))
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm(
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias)
@pytest.mark.parametrize("m,n,k", UNALIGNED_MNK_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm_padded(
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
"""Test CUTLASS FP8 GEMM with padding for non-16-aligned N/K dims.
Exercises CutlassFP8ScaledMMLinearKernel.apply_scaled_mm which pads
inputs to satisfy CUTLASS alignment requirements — the path taken by
models like Qwen2.5-VL whose vision MLP has non-16-aligned dims.
"""
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
CutlassFP8ScaledMMLinearKernel,
)
a = to_fp8(torch.randn((m, k), device="cuda"))
b = to_fp8(torch.randn((n, k), device="cuda").t())
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
scale_a = torch.randn(a_scales_shape, device="cuda", dtype=torch.float32)
scale_b = torch.randn(b_scales_shape, device="cuda", dtype=torch.float32)
scale_a = scale_a.t().contiguous().t()
scale_b = scale_b.t().contiguous().t()
out_dtype = torch.bfloat16
bias = torch.rand((n,), device="cuda", dtype=out_dtype) * 10 if use_bias else None
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
# process_weights_after_loading pad b to 16
pad_k = (16 - k % 16) % 16
pad_n = (16 - n % 16) % 16
if pad_k > 0 or pad_n > 0:
b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t()
if pad_n > 0 and scale_b.numel() > 1:
scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0)
kernel = object.__new__(CutlassFP8ScaledMMLinearKernel)
kernel.logical_output_size = n
out = kernel.apply_scaled_mm(
A=a,
B=b,
out_dtype=out_dtype,
As=scale_a,
Bs=scale_b,
bias=bias,
output_shape=[m, n],
)
torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1)
# Two prefill-sized cases for the SM 12.x blockwise path, where the op switches
# the tile scheduler to a swizzled CTA order once the weight exceeds the L2
# (16384x2560 = 40 MiB and 5120x5120 = 25 MiB on a 24 MiB L2 part); the odd M
# also covers the non-swap-AB dispatch. Elsewhere they run the default order
# like the rest of the list.
BLOCKWISE_PREFILL_FACTORS = [
(8193, 16384, 2560),
(5120, 5120, 5120),
]
@pytest.mark.parametrize("m,n,k", MNK_FACTORS + BLOCKWISE_PREFILL_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))]
)
@pytest.mark.parametrize("use_bias", [False])
@pytest.mark.skipif(
not current_platform.has_device_capability(90),
reason="FP8 blockwise is not supported on this GPU type.",
)
def test_cutlass_fp8_blockwise_scale_gemm(
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
if k % b_scale_group_shape[0] != 0 or n % b_scale_group_shape[1] != 0:
return
if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0:
return
cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
def test_cutlass_int8_gemm(
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
cutlass_int8_gemm_helper(
m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias
)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_bias", [True, False])
def test_cutlass_int8_gemm_output_dtype(
a_scale_group_shape,
b_scale_group_shape,
out_dtype: type[torch.dtype],
use_bias: bool,
):
cutlass_int8_gemm_helper(
512,
512,
512,
a_scale_group_shape,
b_scale_group_shape,
use_bias,
out_dtype=out_dtype,
)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm_output_dtype(
a_scale_group_shape,
b_scale_group_shape,
out_dtype: type[torch.dtype],
use_bias: bool,
):
cutlass_fp8_gemm_helper(
512,
512,
512,
a_scale_group_shape,
b_scale_group_shape,
use_bias,
out_dtype=out_dtype,
)
@pytest.mark.parametrize(
"a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))]
)
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_bias", [False])
@pytest.mark.skipif(
not current_platform.has_device_capability(90),
reason="FP8 blockwise is not supported on this GPU type.",
)
def test_cutlass_fp8_blockwise_scale_gemm_dtype(
a_scale_group_shape,
b_scale_group_shape,
out_dtype: type[torch.dtype],
use_bias: bool,
):
cutlass_fp8_gemm_helper(
512,
512,
512,
a_scale_group_shape,
b_scale_group_shape,
use_bias,
out_dtype=out_dtype,
)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm_devices(
a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str
):
cutlass_fp8_gemm_helper(
512,
512,
512,
a_scale_group_shape,
b_scale_group_shape,
use_bias,
torch.bfloat16,
device,
)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.parametrize("device", CUDA_DEVICES)
def test_cutlass_int8_gemm_devices(
a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str
):
cutlass_int8_gemm_helper(
512,
512,
512,
a_scale_group_shape,
b_scale_group_shape,
use_bias,
out_dtype=torch.bfloat16,
device=device,
)
# For the following two tests:
# N and K correspond to the size of the weight matrix and likely to be multiples
# of a large power of two. In any case, the kernel will have a naive fallback
# when N and K are not divisible by 16. But M is the number of tokens and the
# kernel must handle any M thrown at it.
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.skipif(
not current_platform.has_device_capability(89),
reason="FP8 is not supported on this GPU type.",
)
def test_cutlass_fp8_gemm_m_sweep(
a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
for nk in range(32, 128, 32):
for m in range(1, 128):
cutlass_fp8_gemm_helper(
m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias
)
@pytest.mark.parametrize(
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize(
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
)
@pytest.mark.parametrize("use_bias", [True, False])
def test_cutlass_int8_gemm_m_sweep(
a_scale_group_shape, b_scale_group_shape, use_bias: bool
):
for nk in range(32, 128, 32):
for m in range(1, 128):
cutlass_int8_gemm_helper(
m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias
)
@pytest.mark.parametrize("m", [32, 64, 128])
@pytest.mark.parametrize("n", [16, 32, 64])
@pytest.mark.parametrize("k", [64, 128, 256])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skip
def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int, out_dtype: torch.dtype):
# Currently, the test is failing because folding azp into
# 16-bit bias loses too much precision
scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
aq_i8 = rand_int8((m, k))
bq_i8 = rand_int8((n, k)).t()
aq_i32 = aq_i8.to(dtype=torch.int32)
bq_i32 = bq_i8.to(dtype=torch.int32)
aq_f32 = aq_i8.to(dtype=torch.float32)
bq_f32 = bq_i8.to(dtype=torch.float32)
b_dq = scale_b * bq_f32
azp_a = torch.rand((1,), device="cuda", dtype=torch.float32) * 10 + 1.5
azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
torch.testing.assert_close(a_dq, scale_a * aq_f32 + azp_a)
baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
J = torch.ones((1, k), device="cuda", dtype=torch.float32)
azp_bias = (azp_a * scale_b * (J @ bq_f32)).to(out_dtype)
assert azp_bias.shape == (1, n)
assert azp_bias[0, :].shape == (n,)
baseline_q = (
scale_a.to(device="cpu")
* scale_b.to(device="cpu")
* ((aq_i32 + azp_aq_i8).to(device="cpu") @ bq_i32.to(device="cpu"))
).to(dtype=out_dtype, device="cuda")
out = ops.cutlass_scaled_mm(
aq_i8, bq_i8, scale_a, scale_b, out_dtype=out_dtype, bias=azp_bias[0, :]
)
torch.testing.assert_close(out, baseline_dq, rtol=1e-2, atol=1e0)
torch.testing.assert_close(out, baseline_q, rtol=1e-2, atol=1e0)
@pytest.mark.parametrize("m", [32, 64, 128])
@pytest.mark.parametrize("n", [16, 32, 64])
@pytest.mark.parametrize("k", [64, 128, 256])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.parametrize("azp_per_token", [True, False])
def test_cutlass_int8_azp(
m: int, n: int, k: int, out_dtype: torch.dtype, use_bias: bool, azp_per_token: bool
):
m_azp = m if azp_per_token else 1
scale_a = torch.randn((m_azp, 1), device="cuda", dtype=torch.float32) / 10
scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
aq_i8 = rand_int8((m, k))
aq_i32 = aq_i8.to(dtype=torch.int32)
aq_f32 = aq_i8.to(dtype=torch.float32)
bq_i8 = rand_int8((n, k)).t()
bq_i32 = bq_i8.to(dtype=torch.int32)
bq_f32 = bq_i8.to(dtype=torch.float32)
b_dq = scale_b * bq_f32
azp_a = torch.rand((m_azp, 1), device="cuda", dtype=torch.float32) * 10 + 1.5
azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
torch.testing.assert_close(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
if use_bias:
bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
else:
bias = torch.zeros((1, n), device="cuda", dtype=out_dtype)
baseline_dq = (torch.mm(a_dq, b_dq) + bias).to(out_dtype)
# int32 mm not supported on CUDA
a_noazp_i32_cpu = (aq_i32 - azp_aq_i8).to(device="cpu")
cq = (a_noazp_i32_cpu @ bq_i32.to(device="cpu")).to(device="cuda")
baseline_q = (scale_a * scale_b * cq + bias).to(dtype=out_dtype)
# Hadamard is just the sum of the cols
azp_adj_i32 = bq_i32.sum(dim=0, keepdim=True, dtype=torch.int32)
azp_i32 = azp_aq_i8.to(dtype=torch.int32)
func_bias = bias if use_bias else None
if azp_per_token:
out = ops.cutlass_scaled_mm_azp(
aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_adj_i32, azp_i32, func_bias
)
else:
azp_with_adj_i32 = azp_i32 * azp_adj_i32
out = ops.cutlass_scaled_mm_azp(
aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_with_adj_i32, None, func_bias
)
# bfloat16 precision is 7-bit mantissa -> 2^-8 ~ 0.4%
# float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
atol = 1e-3
torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
if azp_per_token:
opcheck(
torch.ops._C.cutlass_scaled_mm_azp,
(out, aq_i8, bq_i8, scale_a, scale_b, azp_adj_i32, azp_i32, func_bias),
)
else:
opcheck(
torch.ops._C.cutlass_scaled_mm_azp,
(out, aq_i8, bq_i8, scale_a, scale_b, azp_with_adj_i32, None, func_bias),
)
# Test working with a subset of A and B
def test_cutlass_subset():
big_m, big_n, big_k = 1024, 1024, 1024
m, n, k = 512, 512, 512
whole_a = to_int8(torch.randn((big_m, big_k), device="cuda") * 5)
whole_b = to_int8(torch.randn((big_n, big_k), device="cuda").t() * 5)
a = whole_a[0:m, 0:k]
b = whole_b[0:k, 0:n]
scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16)
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16)
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
# Test to make sure cuda graphs work
class CutlassLayer(torch.nn.Module):
def __init__(self, b, scale_a, scale_b, out_dtype):
super().__init__()
self.b = b
self.scale_a = scale_a
self.scale_b = scale_b
self.out_dtype = out_dtype
def forward(self, a):
return ops.cutlass_scaled_mm(
a, self.b, self.scale_a, self.scale_b, self.out_dtype
)
@pytest.mark.parametrize("per_act_token", [True, False])
@pytest.mark.parametrize("per_out_ch", [True, False])
def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
m, n, k = 512, 512, 512
a = to_int8(torch.randn((m, k), device="cuda"))
b = to_int8(torch.randn((n, k), device="cuda").t())
m_a_scales = m if per_act_token else 1
n_b_scales = n if per_out_ch else 1
scale_a = torch.randn((m_a_scales, 1), device="cuda", dtype=torch.float32) / 10
scale_b = torch.randn((1, n_b_scales), device="cuda", dtype=torch.float32) / 10
# Construct a trivial model with a single layer that calls a CUTLASS kernel
model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
# Run the model with a cuda graph
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = model(a)
out.zero_()
g.replay()
baseline = torch.mm(
scale_a * a.to(dtype=torch.float32), scale_b * b.to(dtype=torch.float32)
).to(torch.bfloat16)
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
def test_cutlass_support_opcheck():
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,))
@pytest.mark.parametrize("num_experts", [8, 64])
@pytest.mark.parametrize("per_act_token", [True, False])
@pytest.mark.parametrize("per_out_ch", [True, False])
@pytest.mark.parametrize("use_bias", [False])
@pytest.mark.skipif(
(lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))(
current_platform.get_device_capability()
),
reason="Grouped gemm is not supported on this GPU type.",
)
def test_cutlass_fp8_group_gemm(
num_experts: int, per_act_token: bool, per_out_ch: bool, use_bias: bool
):
# Device and dtype setup
device = "cuda"
out_dtype = torch.half
# Create separate A, B, C tensors for each group
a_tensors = []
b_tensors = []
a_scales_tensors = []
b_scales_tensors = []
baseline_tensors = []
expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int64)
problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32)
if not per_act_token:
one_scale_a = torch.randn((1, 1), device=device, dtype=torch.float32)
alignment = 16 # 128 // 8
# For variation, each group has dimensions
n_g = alignment * random.randint(1, 64)
k_g = alignment * random.randint(1, 64)
for g in range(num_experts):
m_g = alignment * random.randint(1, 64)
expert_offsets[g + 1] = expert_offsets[g] + m_g
problem_sizes[g][0] = m_g
problem_sizes[g][1] = n_g
problem_sizes[g][2] = k_g
m_a_scales = m_g if per_act_token else 1
n_b_scales = n_g if per_out_ch else 1
# Create group-specific A and B (FP8) and output (FP16/FP32)
a_g = to_fp8(torch.randn((m_g, k_g), device=device))
b_g = to_fp8(torch.randn((n_g, k_g), device=device).t())
a_tensors.append(a_g)
b_tensors.append(b_g)
# Set up A/B scales
scale_b = torch.randn((1, n_b_scales), device=device, dtype=torch.float32)
b_scales_tensors.append(scale_b)
if per_act_token:
scale_a = torch.randn((m_a_scales, 1), device=device, dtype=torch.float32)
a_scales_tensors.append(scale_a)
else:
scale_a = one_scale_a
# Compute baseline result for this group
baseline_g = baseline_scaled_mm(a_g, b_g, scale_a, scale_b, out_dtype, None)
baseline_tensors.append(baseline_g)
a_tensors_stacked = torch.empty(
(expert_offsets[num_experts], k_g), device=device, dtype=torch.float8_e4m3fn
)
b_tensors_stacked = torch.empty(
(num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn
)
for g in range(num_experts):
a_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g]
b_tensors_stacked[g] = b_tensors[g].t()
b_tensors_stacked = b_tensors_stacked.transpose(1, 2)
if per_act_token:
a_scales_tensors_stacked = torch.empty(
(expert_offsets[num_experts], 1), device=device, dtype=torch.float32
)
for g in range(num_experts):
a_scales_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = (
a_scales_tensors[g]
)
else:
a_scales_tensors_stacked = one_scale_a
b_scales_tensors_stacked = torch.empty(
(num_experts, n_b_scales), device=device, dtype=torch.float32
)
for g in range(num_experts):
b_scales_tensors_stacked[g] = b_scales_tensors[g]
out_tensors_stacked = torch.zeros(
(expert_offsets[num_experts], n_g), device=device, dtype=out_dtype
)
ab_strides = torch.full(
(num_experts,), a_tensors_stacked.stride(0), device="cuda", dtype=torch.int64
)
c_strides = torch.full(
(num_experts,), out_tensors_stacked.stride(0), device="cuda", dtype=torch.int64
)
ops.cutlass_moe_mm(
out_tensors_stacked,
a_tensors_stacked,
b_tensors_stacked,
a_scales_tensors_stacked,
b_scales_tensors_stacked,
expert_offsets[:-1],
problem_sizes,
ab_strides,
ab_strides,
c_strides,
per_act_token,
per_out_ch,
)
# Validate each group's result against the baseline
for g in range(num_experts):
baseline = baseline_tensors[g]
c = out_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]]
torch.testing.assert_close(c, baseline, rtol=1e-2, atol=5e-4)
@@ -1,328 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the CUTLASS W4A8 kernel.
Run `pytest tests/kernels/quantization/test_cutlass_w4a8.py`.
"""
from dataclasses import dataclass
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.quant_utils import (
convert_packed_uint4b8_to_signed_int4_inplace,
pack_cols,
pack_rows,
quantize_weights,
unpack_quantized_values_into_int32,
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
if not current_platform.is_cuda():
pytest.skip("These tests use CUTLASS which requires CUDA", allow_module_level=True)
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
# unit tests to a common utility function. Currently the use of
# `is_quant_method_supported` conflates kernels with quantization methods
# an assumption which is breaking down as quantizations methods can have
# have kernels and some kernels support multiple quantization methods.
# CUTLASS W4A8 kernels are only built for SM90 (9.0a); the kernel selector
# requires exactly SM90 too, so match it here rather than accepting >= 90.
IS_SUPPORTED_BY_GPU = (
current_platform.is_cuda() and current_platform.is_device_capability(90)
)
MNK_SHAPES = [
(1, 128, 128),
(1, 512, 1024),
(1, 4096, 4096),
(1, 8192, 28672),
(13, 8192, 4096),
(26, 4096, 8192),
(64, 4096, 4096),
(64, 8192, 28672),
(257, 128, 4096),
(257, 4096, 4096),
(1024, 4096, 8192),
(1024, 8192, 4096),
]
# TODO(czhu): get supported schedules from fn
SCHEDULES = [
"128x16_1x1x1",
"256x16_1x1x1",
"128x32_1x1x1",
"256x32_1x1x1",
"128x64_1x1x1",
"256x64_1x1x1",
"128x128_1x1x1",
"256x128_1x1x1",
"128x256_1x1x1",
"128x256_2x1x1",
]
@dataclass
class TypeConfig:
act_type: torch.dtype
weight_type: ScalarType
output_type: torch.dtype | None
group_scale_type: torch.dtype | None
channel_scale_type: torch.dtype | None
token_scale_type: torch.dtype | None
@dataclass
class Tensors:
w_ref: torch.Tensor
a_ref: torch.Tensor
a: torch.Tensor
w_q: torch.Tensor
w_g_s: torch.Tensor
w_ch_s: torch.Tensor
w_tok_s: torch.Tensor
# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
# Ch Scales Type, Tok Scales Type)
TestTypeTuple = tuple[
list[torch.dtype], ScalarType, torch.dtype | None, torch.dtype | None, bool
]
TEST_TYPES = [
*(
TypeConfig(
act_type=torch.float8_e4m3fn,
weight_type=w_type,
output_type=o_type,
group_scale_type=torch.float8_e4m3fn,
channel_scale_type=torch.float32,
token_scale_type=torch.float32,
)
for w_type in [scalar_types.int4]
# TODO(czhu): fp16 out type
for o_type in [torch.bfloat16]
),
]
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
# unit tests to a common utility function. Currently the use of
# For testing quantized linear kernels
def to_fp8(tensor: torch.Tensor):
finfo = torch.finfo(torch.float8_e4m3fn)
return tensor.clamp(min=finfo.min, max=finfo.max).to(dtype=torch.float8_e4m3fn)
def cutlass_quantize_and_pack(
atype: torch.dtype,
w: torch.Tensor,
wtype: ScalarType,
stype: torch.dtype | None,
group_size: int | None,
zero_points: bool = False,
):
assert wtype.is_integer(), "TODO: support floating point weights"
w_ref, w_q, w_s, w_zp = quantize_weights(
w, wtype, group_size=group_size, zero_points=zero_points
)
# since scales are cast to fp8, we need to compute w_ref this way
w_ref = (
(w_q).to(torch.float32)
* w_s.to(atype).to(torch.float32).repeat_interleave(group_size, dim=0)
).to(atype)
# bit mask prevents sign extending int4 when packing
w_q = pack_rows(w_q & 0x0F, wtype.size_bits, *w_q.shape)
w_q = w_q.t().contiguous().t() # convert to col major
w_q_packed = ops.cutlass_encode_and_reorder_int4b(w_q)
w_s_packed = ops.cutlass_pack_scale_fp8(w_s.to(atype))
return w_ref, w_q_packed, w_s_packed, w_zp
def create_test_tensors(
shape: tuple[int, int, int], types: TypeConfig, group_size: int | None
) -> Tensors:
m, n, k = shape
print(
"create_test_tensors, shape:", shape, "types:", types, "group_size:", group_size
)
a = to_fp8(torch.randn((m, k), device="cuda"))
w = to_fp8(torch.randn((k, n), device="cuda"))
if types.group_scale_type is not None:
w = w.to(types.group_scale_type)
if w.dtype.itemsize == 1:
w = w.to(torch.float16)
w_ref, w_q_packed, w_s, _ = cutlass_quantize_and_pack(
a.dtype, w, types.weight_type, types.group_scale_type, group_size, False
)
a_ref = a.to(torch.float32)
w_ref = w_ref.to(torch.float32)
# for the practical use case we need per-tok scales for fp8 activations
w_tok_s = torch.randn((m,), device="cuda", dtype=types.token_scale_type)
w_ch_s = torch.randn((n,), device="cuda", dtype=types.channel_scale_type)
return Tensors(
w_ref=w_ref,
a_ref=a_ref,
a=a,
w_q=w_q_packed,
w_g_s=w_s,
w_ch_s=w_ch_s,
w_tok_s=w_tok_s,
)
def mm_test_helper(
types: TypeConfig,
tensors: Tensors,
group_size: int | None = None,
schedule: str | None = None,
):
# CUTLASS upstream uses fp8 with fastaccum as reference
# https://github.com/NVIDIA/cutlass/blob/main/examples/55_hopper_mixed_dtype_gemm/55_hopper_int4_fp8_gemm.cu#L406
output_ref = torch._scaled_mm(
tensors.a_ref.to(types.act_type),
tensors.w_ref.to(types.act_type).t().contiguous().t(), # col major
tensors.w_tok_s.unsqueeze(1),
tensors.w_ch_s.unsqueeze(0),
out_dtype=types.output_type,
use_fast_accum=True,
)
output = ops.cutlass_w4a8_mm(
a=tensors.a,
b_q=tensors.w_q,
b_group_scales=tensors.w_g_s,
b_group_size=group_size,
b_channel_scales=tensors.w_ch_s,
a_token_scales=tensors.w_tok_s,
)
print(output)
print(output_ref)
torch.testing.assert_close(
output, output_ref.to(output.dtype), rtol=1e-2, atol=1e-2
)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
)
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
@pytest.mark.parametrize("types", TEST_TYPES)
@pytest.mark.parametrize("schedule", SCHEDULES)
def test_cutlass_w4a8(shape, types: TypeConfig, schedule):
group_sizes = [128]
for group_size in group_sizes:
tensors = create_test_tensors(shape, types, group_size)
mm_test_helper(types, tensors, group_size, schedule)
# Test to make sure cuda graphs work
class W4A8Layer(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
def forward(self, a):
return ops.cutlass_w4a8_mm(a=a, **self.kwargs)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
)
def test_w4a8_cuda_graph():
m, n, k = 512, 4096, 4096
a = to_fp8(torch.randn((m, k), device="cuda"))
b = to_fp8(torch.randn((k, n), device="cuda"))
wtype = scalar_types.int4
stype = torch.float8_e4m3fn
group_size = 128
zero_points = False
w_ref, w_q_packed, w_s, _ = cutlass_quantize_and_pack(
a.dtype, b.to(torch.float16), wtype, stype, group_size, zero_points
)
w_tok_s = torch.randn((m,), device="cuda", dtype=torch.float32)
w_ch_s = torch.randn((n,), device="cuda", dtype=torch.float32)
# Construct a trivial model with a single layer that calls the kernel
model = W4A8Layer(
b_q=w_q_packed,
b_group_scales=w_s,
b_group_size=group_size,
b_channel_scales=w_ch_s,
a_token_scales=w_tok_s,
)
output_ref = torch._scaled_mm(
a,
w_ref.to(a.dtype).t().contiguous().t(), # col major
w_tok_s.unsqueeze(1),
w_ch_s.unsqueeze(0),
out_dtype=torch.bfloat16,
use_fast_accum=True,
)
# Run the model with a cuda graph
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
output = model(a)
output.zero_()
g.replay()
torch.testing.assert_close(output, output_ref, rtol=1e-2, atol=1e-2)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
)
@pytest.mark.parametrize("shape", MNK_SHAPES)
def test_convert_packed_uint4b8_to_signed_int4_inplace(shape):
"""The W4A16 checkpoints encode the weights as int4b8 packed to int32.
The CUTLASS kernels expect signed int4 packed to int32.
This tests checks that the runtime int4b8 -> signed int4 conversion
matches the offline conversion step exactly.
"""
_, N, K = shape
# random weights packed to int32
t = torch.randint(
low=torch.iinfo(torch.int32).min,
high=torch.iinfo(torch.int32).max + 1,
size=(N, K // 8),
dtype=torch.int32,
device="cuda",
)
# compute reference
unpacked = unpack_quantized_values_into_int32(
t.clone(), scalar_types.uint4b8, packed_dim=1
)
unpacked = unpacked - 8 # int4b8 -> signed int4
ref = pack_cols(unpacked & 0x0F, 4, *unpacked.shape)
out = convert_packed_uint4b8_to_signed_int4_inplace(t.clone())
assert torch.equal(ref, out)
assert not torch.equal(ref, t)
@@ -1,337 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the CUTLASS-based W4A8 grouped GEMM kernel and the full MoE layer."""
import random
from dataclasses import dataclass
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.quant_utils import (
pack_rows,
quantize_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
from vllm.utils.torch_utils import set_random_seed
IS_SUPPORTED_BY_GPU = (
current_platform.is_cuda() and current_platform.is_device_capability(90)
)
def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
finfo = torch.finfo(torch.float8_e4m3fn)
return tensor.clamp(min=finfo.min, max=finfo.max).to(dtype=torch.float8_e4m3fn)
def cutlass_quantize(
atype: torch.dtype,
w: torch.Tensor,
wtype: ScalarType,
stype: torch.dtype | None,
group_size: int | None,
zero_points: bool = False,
):
"""Quantize weights into W4 and compute reference dequantized weights.
Encoding/reordering of weights and packing of scales is deferred
until after all experts are combined.
"""
assert wtype.is_integer(), "TODO: support floating point weights"
w_ref, w_q, w_s, w_zp = quantize_weights(
w, wtype, group_size=group_size, zero_points=zero_points
)
# Since scales are later cast to fp8, recompute w_ref in atype here.
w_ref = (
w_q.to(torch.float32)
* w_s.to(atype).to(torch.float32).repeat_interleave(group_size, dim=0)
).to(atype)
# Bit mask prevents sign extension of int4 when packing.
w_q = pack_rows(w_q & 0x0F, wtype.size_bits, *w_q.shape)
# Make weights row-major (N, K).
w_q = w_q.t().contiguous()
return w_ref, w_q, w_s.to(atype), w_zp
def cutlass_preprocess(
w_q_experts: list[torch.Tensor], w_s_experts: list[torch.Tensor]
):
"""Reorder/encode expert weights and pack scales.
Returns:
w_q_packed: Packed/encoded int4 weights for all experts.
w_s_packed: Packed fp8 scales for all experts.
packed_layout: Layout/stride metadata for grouped GEMM.
"""
w_s_packed = ops.cutlass_pack_scale_fp8(torch.stack(w_s_experts))
w_q_packed, packed_layout = ops.cutlass_encode_and_reorder_int4b_grouped(
torch.stack(w_q_experts)
) # expects dim 3
return w_q_packed, w_s_packed, packed_layout
GROUP_SIZE = 128
# (num_experts, N, K)
TEST_SHAPES = [
(8, 512, 2048),
(8, 2048, 2048),
(64, 512, 1024),
(64, 2048, 2048),
(4, 2048, 768),
(8, 768, 2048),
(64, 1536, 2048),
(128, 8192, 4096), # test overflow int32
]
ALIGNMENT = 16 # torch._scaled_mm alignment for M, needed for reference check
@dataclass
class MoETestSetup:
num_experts: int
K: int
N: int
Ms: list[int]
M_full: int
a: torch.Tensor
a_ref: torch.Tensor
a_strides: torch.Tensor
out: torch.Tensor
c_strides: torch.Tensor
per_tok_scales: torch.Tensor
per_chan_scales: torch.Tensor
w_refs: list[torch.Tensor]
w_q_packed: torch.Tensor
w_s_packed: torch.Tensor
problem_sizes: torch.Tensor
expert_offsets: torch.Tensor
b_strides: torch.Tensor
group_scale_strides: torch.Tensor
def make_moe_test_setup(
num_experts: int,
K: int,
N: int,
*,
alignment: int = ALIGNMENT,
max_blocks: int = 64,
device: str = "cuda",
random_zero: bool = False,
) -> MoETestSetup:
"""Create a full set of tensors for testing cutlass_w4a8_moe_mm."""
assert K % GROUP_SIZE == 0
# Token counts per expert (multiples of `alignment`).
Ms = [alignment * random.randint(1, max_blocks) for _ in range(num_experts)]
# set random experts to 0 tokens
if random_zero and num_experts > 1:
num_zero = max(1, num_experts // 8)
zero_indices = random.sample(range(num_experts), k=num_zero)
for idx in zero_indices:
Ms[idx] = 0
M_full = sum(Ms)
assert M_full > 0
# Activations.
a = to_fp8(torch.randn((M_full, K), device=device))
a_ref = a.to(torch.float32)
a_strides = torch.full((num_experts,), K, dtype=torch.int64, device=device)
# Output buffer.
out = torch.empty((M_full, N), dtype=torch.bfloat16, device=device)
c_strides = torch.full((num_experts,), N, dtype=torch.int64, device=device)
# Channel/token scales.
per_tok_scales = torch.randn((M_full, 1), dtype=torch.float32, device=device)
per_chan_scales = torch.randn(
(num_experts, N, 1), dtype=torch.float32, device=device
)
# Expert weights and scales.
wtype = scalar_types.int4
atype = stype = torch.float8_e4m3fn
w_refs, w_qs, w_ss = [], [], []
for _ in range(num_experts):
b = to_fp8(torch.randn((K, N), device=device))
w_ref, w_q, w_s, _ = cutlass_quantize(
atype, b.to(torch.float16), wtype, stype, GROUP_SIZE, zero_points=False
)
w_refs.append(w_ref)
w_qs.append(w_q)
w_ss.append(w_s)
w_q_packed, w_s_packed, packed_layout = cutlass_preprocess(w_qs, w_ss)
problem_sizes = torch.tensor(
[[N, M, K] for M in Ms], dtype=torch.int32, device=device
)
expert_offsets = torch.cat(
[
torch.tensor([0], dtype=torch.int64),
torch.cumsum(torch.tensor(Ms, dtype=torch.int64), dim=0)[:-1],
]
).to(device=device)
# B strides and group scale strides.
b_strides = packed_layout
group_scale_strides = torch.zeros(
(num_experts, 2), dtype=torch.int64, device=device
)
group_scale_strides[:, 0] = N
return MoETestSetup(
num_experts=num_experts,
K=K,
N=N,
Ms=Ms,
M_full=M_full,
a=a,
a_ref=a_ref,
a_strides=a_strides,
out=out,
c_strides=c_strides,
per_tok_scales=per_tok_scales,
per_chan_scales=per_chan_scales,
w_refs=w_refs,
w_q_packed=w_q_packed,
w_s_packed=w_s_packed,
problem_sizes=problem_sizes,
expert_offsets=expert_offsets,
b_strides=b_strides,
group_scale_strides=group_scale_strides,
)
def compute_moe_reference_output(setup: MoETestSetup) -> torch.Tensor:
"""Compute reference output using torch._scaled_mm per expert."""
out_ref = torch.empty_like(setup.out)
ends = torch.cumsum(torch.tensor(setup.Ms), 0).tolist()
starts = setup.expert_offsets.cpu().tolist()
for i in range(setup.num_experts):
start, end = starts[i], ends[i]
if start == end:
continue
out_ref_i = torch._scaled_mm(
setup.a_ref[start:end].to(torch.float8_e4m3fn),
setup.w_refs[i].to(torch.float8_e4m3fn).t().contiguous().t(),
setup.per_tok_scales[start:end], # (M, 1)
setup.per_chan_scales[i].reshape(1, -1), # (1, N)
out_dtype=torch.bfloat16,
use_fast_accum=True,
)
out_ref[start:end] = out_ref_i
return out_ref
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU,
reason="W4A8 Grouped GEMM is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", TEST_SHAPES)
@pytest.mark.parametrize("random_zero", [True, False])
def test_cutlass_w4a8_moe_mm_end_to_end(shape, random_zero):
num_experts, N, K = shape
set_random_seed(42)
setup = make_moe_test_setup(
num_experts=num_experts, K=K, N=N, max_blocks=64, random_zero=random_zero
)
ops.cutlass_w4a8_moe_mm(
setup.out,
setup.a,
setup.w_q_packed,
setup.per_tok_scales,
setup.per_chan_scales,
setup.w_s_packed,
GROUP_SIZE,
setup.expert_offsets,
setup.problem_sizes,
setup.a_strides,
setup.b_strides,
setup.c_strides,
setup.group_scale_strides,
)
torch.accelerator.synchronize()
out_ref = compute_moe_reference_output(setup)
torch.testing.assert_close(setup.out, out_ref, rtol=1e-2, atol=1e-2)
class W4A8MoELayer(torch.nn.Module):
"""Minimal wrapper module to test cuda graphs."""
def __init__(self, setup: MoETestSetup):
super().__init__()
self.setup = setup
def forward(self, a: torch.Tensor) -> torch.Tensor:
s = self.setup
ops.cutlass_w4a8_moe_mm(
s.out,
a,
s.w_q_packed,
s.per_tok_scales,
s.per_chan_scales,
s.w_s_packed,
GROUP_SIZE,
s.expert_offsets,
s.problem_sizes,
s.a_strides,
s.b_strides,
s.c_strides,
s.group_scale_strides,
)
return s.out
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU,
reason="W4A8 Grouped GEMM is not supported on this GPU type.",
)
def test_cutlass_w4a8_moe_mm_cuda_graph():
set_random_seed(42)
# Fixed config for CUDA graph test (single parameter point).
num_experts = 8
K = 512
N = 2048
setup = make_moe_test_setup(
num_experts=num_experts,
K=K,
N=N,
max_blocks=32,
)
# Construct model that calls the grouped GEMM kernel.
model = W4A8MoELayer(setup)
# Build reference output once.
out_ref = compute_moe_reference_output(setup)
# Capture and run the model in a CUDA graph.
a_static = setup.a.clone() # static input tensor for graph replay
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out_static = model(a_static)
out_static.zero_()
g.replay()
torch.testing.assert_close(out_static, out_ref, rtol=1e-2, atol=1e-2)
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.kernels.linear import (
FlashInferTrtllmMxfp8LinearKernel,
Mxfp8LinearLayerConfig,
)
from vllm.platforms import current_platform
from vllm.utils import flashinfer as vllm_flashinfer
from vllm.utils.flashinfer import has_flashinfer
if not (
current_platform.is_cuda()
and current_platform.is_device_capability_family(100)
and has_flashinfer()
):
pytest.skip(
reason="FlashInfer TRTLLM MXFP8 requires an SM100-family GPU",
allow_module_level=True,
)
def _make_layer(weight: torch.Tensor) -> torch.nn.Module:
from flashinfer import SfLayout, mxfp8_quantize
weight_mxfp8, weight_scale = mxfp8_quantize(
weight,
sf_swizzle_layout=SfLayout.layout_linear,
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(weight_mxfp8, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(
weight_scale.view(weight.shape[0], weight.shape[1] // 32),
requires_grad=False,
)
return layer
@pytest.mark.parametrize("shape", [(1, 130, 256), (7, 256, 512), (128, 130, 768)])
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_linear_numerics(
shape: tuple[int, int, int],
) -> None:
torch.manual_seed(0)
m, n, k = shape
x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(
Mxfp8LinearLayerConfig(weight_shape=(n, k))
)
kernel.process_weights_after_loading(layer)
output = kernel.apply_weights(layer, x)
reference = torch.mm(x, weight.t())
similarity = F.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
)
assert output.shape == (m, n)
assert output.is_contiguous()
assert similarity.item() > 0.98
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_custom_ops() -> None:
x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda")
weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(
Mxfp8LinearLayerConfig(weight_shape=(256, 512))
)
kernel.process_weights_after_loading(layer)
torch.library.opcheck(
torch.ops.vllm.flashinfer_mxfp8_quantize_8x4.default,
(x,),
)
x_mxfp8, x_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(x)
# SchemaCheckMode compares inputs with allclose, which CUDA does not
# implement for float8. The numerical tests above guard input mutation.
torch.library.opcheck(
torch.ops.vllm.mm_mxfp8.default,
(
x_mxfp8,
layer.weight.t(),
x_scale,
layer.weight_scale,
torch.bfloat16,
"trtllm",
True,
),
test_utils=(
"test_autograd_registration",
"test_faketensor",
"test_aot_dispatch_dynamic",
),
)
@torch.inference_mode()
def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None:
torch.manual_seed(0)
m, n, k = 7, 130, 512
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
layer = _make_layer(weight)
kernel = FlashInferTrtllmMxfp8LinearKernel(
Mxfp8LinearLayerConfig(weight_shape=(n, k))
)
kernel.process_weights_after_loading(layer)
static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
kernel.apply_weights(layer, static_x)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_output = kernel.apply_weights(layer, static_x)
new_x = torch.randn_like(static_x)
static_x.copy_(new_x)
graph.replay()
eager_output = kernel.apply_weights(layer, new_x)
torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0)
@@ -1,280 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
convert_swizzled_to_linear,
dequantize_nvfp4_to_dtype,
)
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.nvfp4 import NvFp4LinearLayerConfig
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCuteDslNvFp4LinearKernel,
FlashInferCuteDslNvFp4W4A16LinearKernel,
FlashInferCutlassNvFp4LinearKernel,
)
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.flashinfer import (
flashinfer_scaled_fp4_mm,
has_flashinfer_b12x_gemm,
)
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
reason="Nvfp4 Requires compute capability of 10 or above.",
allow_module_level=True,
)
DTYPES = [torch.float16, torch.bfloat16]
# m, n, k
SHAPES = [
(128, 128, 64),
(128, 128, 128),
(256, 128, 64),
(128, 256, 128),
(1, 128, 128),
]
PAD_SHAPES = [(150, 128, 64), (128, 128, 96), (2, 128, 64), (3, 128, 96)]
SHAPES.extend(PAD_SHAPES)
SEEDS = [42]
CUDA_DEVICES = ["cuda:0"]
@pytest.mark.parametrize("token_shape", [(17,), (1, 17), (2, 17)])
@pytest.mark.parametrize(
"kernel_cls",
[FlashInferCutlassNvFp4LinearKernel, FlashInferCuteDslNvFp4LinearKernel],
)
@torch.inference_mode()
def test_flashinfer_nvfp4_preserves_quantized_input_shape(
token_shape: tuple[int, ...],
kernel_cls: type[FlashInferCutlassNvFp4LinearKernel]
| type[FlashInferCuteDslNvFp4LinearKernel],
) -> None:
"""Pre-quantized inputs must match the tensor path and retain token dimensions."""
supported, reason = kernel_cls.is_supported()
if not supported:
pytest.skip(reason)
set_random_seed(42)
hidden_size, output_size = 128, 128
x = torch.randn(*token_shape, hidden_size, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(output_size, hidden_size, dtype=x.dtype, device=x.device)
input_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / x.abs().max()).float()
weight_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).float()
x_fp4, x_blockscale = ops.scaled_fp4_quant(x.reshape(-1, hidden_size), input_scale)
weight_fp4, weight_blockscale = ops.scaled_fp4_quant(weight, weight_scale)
quantized_input = QuantizedActivation(
data=x_fp4.reshape(*token_shape, hidden_size // 2),
scale=x_blockscale,
orig_dtype=x.dtype,
orig_shape=x.shape,
quant_key=kNvfp4Dynamic,
)
layer = torch.nn.Module()
layer.input_size_per_partition = hidden_size
layer.output_size_per_partition = output_size
layer.weight = weight_fp4
layer.weight_scale = weight_blockscale
layer.input_global_scale_inv = input_scale
layer.alpha = (input_scale * weight_scale).reciprocal()
kernel = kernel_cls(NvFp4LinearLayerConfig())
expected = kernel.apply_weights(layer, x.reshape(-1, hidden_size))
actual = kernel.apply_weights(layer, quantized_input)
assert actual.shape == (*token_shape, output_size)
assert actual.dtype == x.dtype
torch.testing.assert_close(actual.reshape(expected.shape), expected)
def get_ref_results(
a_fp4,
b_fp4,
a_sf,
b_sf,
a_global_scale,
b_global_scale,
m,
n,
dtype,
block_size,
device,
is_sf_128x4_layout,
):
_, m_k = a_fp4.shape
_, n_k = b_fp4.shape
assert m_k == n_k
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4,
a_sf,
a_global_scale,
dtype=dtype,
device=device,
block_size=block_size,
is_sf_128x4_layout=is_sf_128x4_layout,
)
b_in_dtype = dequantize_nvfp4_to_dtype(
b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size
)
return torch.matmul(a_in_dtype, b_in_dtype.t())
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass", "cudnn", "trtllm", "b12x"])
@pytest.mark.parametrize("autotune", [False, True])
@torch.inference_mode()
def test_flashinfer_nvfp4_gemm(
dtype: torch.dtype,
shape: tuple[int, int, int],
seed: int,
device: str,
backend: str,
autotune: bool,
) -> None:
if "trtllm" in backend and dtype == torch.float16:
pytest.skip("Only torch.bfloat16 is supported for TRTLLM FP4 GEMM operations")
if backend == "cute-dsl" and not current_platform.is_device_capability_family(100):
pytest.skip("FlashInfer cutedsl backend is only supported on SM10x")
if backend == "b12x" and not current_platform.has_device_capability(120):
pytest.skip("b12x FP4 GEMM requires SM120+ (CC 12.0+)")
if backend == "b12x" and not has_flashinfer_b12x_gemm():
pytest.skip("b12x FP4 GEMM backend not available in installed FlashInfer")
set_random_seed(seed)
m, n, packed_k = shape
k = packed_k * 2
block_size = 16
a_dtype = torch.randn((m, k), dtype=dtype, device=device)
b_dtype = torch.randn((n, k), dtype=dtype, device=device)
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1)
).to(torch.float32)
b_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1)
).to(torch.float32)
alpha = 1.0 / (a_global_scale * b_global_scale)
# ops.scaled_fp4_quant returns swizzled scales, while weights
# from checkpoints are in linear scales.
# cutlass and b12x use swizzled scales directly; trtllm needs them unswizzled.
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(
a_dtype, a_global_scale, is_sf_swizzled_layout=True, backend=backend
)
is_sf_128x4_layout = not (backend == "trtllm" and m <= 32)
b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(
b_dtype, b_global_scale, is_sf_swizzled_layout=True
)
# get_ref_results unswizzles the scales internally.
expected_out = get_ref_results(
a_fp4,
b_fp4,
a_scale_interleaved,
b_scale_interleaved,
a_global_scale,
b_global_scale,
m,
n,
dtype,
block_size,
device,
is_sf_128x4_layout,
)
import flashinfer
if "trtllm" in backend:
epilogue_tile_m = 128
b_fp4 = flashinfer.shuffle_matrix_a(b_fp4.view(torch.uint8), epilogue_tile_m)
b_scale_interleaved = convert_swizzled_to_linear(
b_scale_interleaved, n, k, block_size
)
b_scale_interleaved = (
flashinfer.shuffle_matrix_sf_a(
b_scale_interleaved.view(torch.uint8), epilogue_tile_m
)
.reshape(b_scale_interleaved.shape)
.view(torch.float8_e4m3fn)
)
with flashinfer.autotune(autotune):
out = flashinfer_scaled_fp4_mm(
a_fp4,
b_fp4,
a_scale_interleaved,
b_scale_interleaved,
alpha,
dtype,
backend=backend,
)
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_flashinfer_cutedsl_nvfp4_w4a16_linear(
shape: tuple[int, int, int],
seed: int,
device: str,
) -> None:
supported, reason = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported()
if not supported:
pytest.skip(reason)
set_random_seed(seed)
m, n, packed_k = shape
k = packed_k * 2
x = torch.randn((m, k), dtype=torch.bfloat16, device=device)
weight = torch.randn((n, k), dtype=torch.bfloat16, device=device)
weight_quant_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).to(
torch.float32
)
weight_global_scale = weight_quant_scale.reciprocal()
weight_fp4, weight_scale = ops.scaled_fp4_quant(
weight,
weight_quant_scale,
is_sf_swizzled_layout=False,
)
weight_ref = dequantize_to_dtype(
weight_fp4,
weight_scale,
weight_global_scale,
dtype=torch.bfloat16,
swizzle=False,
)
layer = torch.nn.Module()
layer.output_size_per_partition = n
layer.weight = torch.nn.Parameter(weight_fp4, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
layer.weight_global_scale = torch.nn.Parameter(
weight_global_scale, requires_grad=False
)
kernel = FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearLayerConfig())
kernel.process_weights_after_loading(layer)
output = kernel.apply_weights(layer, x)
expected = F.linear(x, weight_ref)
assert output.shape == (m, n)
torch.testing.assert_close(output, expected, atol=1e-1, rtol=1e-1)
@@ -1,73 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.flashinfer import flashinfer_scaled_fp8_mm
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
reason="Flashinfer FP8 gemms requires compute capability of 10.0 or above.",
allow_module_level=True,
)
DTYPES = [torch.float16, torch.bfloat16]
# m, n, k
SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)]
PAD_SHAPES = [(150, 128, 64), (128, 128, 96)]
SHAPES.extend(PAD_SHAPES)
SEEDS = [42]
CUDA_DEVICES = ["cuda:0"]
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("use_bias", [True, False])
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("autotune", [False, True])
@torch.inference_mode()
def test_flashinfer_fp8_gemm(
dtype: torch.dtype,
shape: tuple[int, int, int],
use_bias: bool,
seed: int,
device: str,
autotune: bool,
) -> None:
set_random_seed(seed)
m, n, k = shape
a = torch.randn((m, k), dtype=dtype, device=device)
b = torch.randn((n, k), dtype=dtype, device=device) / k
a_fp8, a_scale = ops.scaled_fp8_quant(a)
b_fp8, b_scale = ops.scaled_fp8_quant(b)
expected_out = torch.mm(
a_scale * a_fp8.to(dtype=torch.float32),
b_scale * b_fp8.to(dtype=torch.float32).t(),
).to(dtype=dtype)
if use_bias:
bias = torch.randn((n,), dtype=dtype, device=device)
expected_out = expected_out + bias
else:
bias = None
import flashinfer
with flashinfer.autotune(autotune):
out = flashinfer_scaled_fp8_mm(
a_fp8,
b_fp8.t(),
a_scale,
b_scale,
dtype,
bias=bias,
)
torch.testing.assert_close(out, expected_out, atol=1e-2, rtol=1e-2)
@@ -1,255 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import (
FP8_DTYPE,
ref_dynamic_per_tensor_fp8_quant,
ref_dynamic_per_token_quant,
)
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
scaled_quantize,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
DTYPES = [torch.bfloat16, torch.float]
HIDDEN_SIZES = [17, 1024, 1025, 1026, 5137, 8193]
NUM_TOKENS = [1, 7, 4096]
SCALE_UBS = [True, False]
SEEDS = [0]
def opcheck_fp8_quant(
output,
input,
scale=None,
scale_ub=None,
use_per_token_if_dynamic=False,
group_shape=None,
):
if scale is not None:
opcheck(
torch.ops._C.static_scaled_fp8_quant,
(output, input, scale, group_shape),
)
elif use_per_token_if_dynamic:
scale = torch.empty(
(input.shape[0], 1), device=input.device, dtype=torch.float32
)
opcheck(
torch.ops._C.dynamic_per_token_scaled_fp8_quant,
(output, input, scale, scale_ub),
)
else:
scale = torch.empty(
(input.numel() // input.shape[-1], 1),
device=input.device,
dtype=torch.float32,
)
opcheck(torch.ops._C.dynamic_scaled_fp8_quant, (output, input, scale))
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("do_scale_ub", SCALE_UBS)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_dynamic_per_token_fp8_quant(
num_tokens: int, hidden_size: int, dtype: torch.dtype, do_scale_ub: bool, seed: int
) -> None:
set_random_seed(seed)
x = (
torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") + 1e-6
) # avoid nans
scale_ub = (
torch.mean(x).to(dtype=torch.float32, device="cuda") if do_scale_ub else None
)
ref_out, ref_scales = ref_dynamic_per_token_quant(x, FP8_DTYPE, scale_ub)
ops_out, ops_scales = ops.scaled_fp8_quant(
x, scale_ub=scale_ub, use_per_token_if_dynamic=True
)
torch.testing.assert_close(ref_scales, ops_scales)
torch.testing.assert_close(
ref_out.to(dtype=torch.float32), ops_out.to(dtype=torch.float32)
)
opcheck_fp8_quant(ops_out, x, None, scale_ub, use_per_token_if_dynamic=True)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_dynamic_per_tensor_fp8_quant(
num_tokens: int, hidden_size: int, dtype: torch.dtype, seed: int
) -> None:
set_random_seed(seed)
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
ops_out, ops_scale = ops.scaled_fp8_quant(x)
torch.testing.assert_close(ref_scale, ops_scale)
torch.testing.assert_close(
ref_out.to(dtype=torch.float32), ops_out.to(dtype=torch.float32)
)
opcheck_fp8_quant(ops_out, x)
@pytest.mark.parametrize("pad", [8, 16])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_dynamic_per_tensor_fp8_quant_strided_input(
pad: int, dtype: torch.dtype, seed: int
) -> None:
"""Non-contiguous rows take the strided abs-max kernel; contiguous rows take
the flat one. Both must produce exactly the same scale and bytes.
pad=8 (16 B for bf16) leaves rows misaligned for 32 B vector loads, pad=16
keeps them aligned but non-contiguous.
"""
set_random_seed(seed)
num_tokens, hidden_size = 64, 4096
full = torch.rand(num_tokens, hidden_size + pad, dtype=dtype, device="cuda")
x = full[:, :hidden_size]
assert not x.is_contiguous()
ops_out, ops_scale = ops.scaled_fp8_quant(x)
flat_out, flat_scale = ops.scaled_fp8_quant(x.contiguous())
assert torch.equal(ops_scale, flat_scale)
assert torch.equal(ops_out.view(torch.uint8), flat_out.view(torch.uint8))
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
assert torch.equal(ref_scale, ops_scale)
assert torch.equal(ref_out.view(torch.uint8), ops_out.view(torch.uint8))
# Regression test for a case with large activations where an int32 index cannot
# represent the number of elements.
@torch.inference_mode()
@pytest.mark.parametrize("seed", SEEDS)
def test_fp8_quant_large(seed: int) -> None:
set_random_seed(seed)
num_tokens = 1024000 # Mistral-Nemo's max_position_embeddings
hidden_size = 1152 # Smallest hidden_size to reproduce the error
dtype = torch.bfloat16
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, scale = ref_dynamic_per_tensor_fp8_quant(x)
ops_out, _ = ops.scaled_fp8_quant(x, scale)
# Minimize memory footprint in this test by freeing x and upconverting
# the outputs in place. (torch.allclose does not support fp8)
del x
ref_out = ref_out.to(dtype=dtype)
ops_out = ops_out.to(dtype=dtype)
torch.testing.assert_close(ref_out, ops_out)
# Test static FP8 quantization with 2D group scales
GROUP_SHAPES_2D = [
(-1, -1), # Per-tensor
(-1, 1), # Per-channel
(1, -1), # Per-token
(-1, 128), # Per-head quantization
(1, 128), # DeepSeek-style per-token-per-group (group_m=1, group_n=128)
(128, 128), # DeepSeek-style block quantization
(1, 64), # Smaller group size
(1, 16), # Small group (scalar path in kernel)
(4, 256), # Non-trivial both dimensions
]
# Use sizes divisible by all group shapes
NUM_TOKENS_GROUP = [128, 512]
HIDDEN_SIZES_GROUP = [256, 1024, 2048]
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_GROUP)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES_GROUP)
@pytest.mark.parametrize("group_shape", GROUP_SHAPES_2D)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@torch.inference_mode()
def test_static_fp8_quant_group_2d(
num_tokens: int,
hidden_size: int,
group_shape: tuple[int, int],
dtype: torch.dtype,
seed: int,
) -> None:
"""Test static FP8 quantization with 2D group scales using scaled_quantize."""
# Normalize group_shape (-1 means full extent)
norm_group_m = num_tokens if group_shape[0] == -1 else group_shape[0]
norm_group_n = hidden_size if group_shape[1] == -1 else group_shape[1]
# Skip if sizes are not divisible by group shape
if num_tokens % norm_group_m != 0 or hidden_size % norm_group_n != 0:
pytest.skip(
f"Skipping: ({num_tokens}, {hidden_size}) not divisible by "
f"group_shape ({group_shape[0]}, {group_shape[1]})"
)
set_random_seed(seed)
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, scale = scaled_quantize(
x,
GroupShape(*group_shape),
current_platform.fp8_dtype(),
compute_dtype=torch.float32,
)
ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape)
torch.testing.assert_close(scale, ops_scale)
torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=1.2e-1, atol=1e-5)
opcheck_fp8_quant(ops_out, x, scale=scale)
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_GROUP)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES_GROUP)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("group_shape", [(1, -1), (-1, 1)]) # per-token, per-channel
@torch.inference_mode()
def test_static_fp8_quant_1d_scale(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
seed: int,
group_shape: tuple[int, int],
) -> None:
"""Test static FP8 quantization with 1D scale (per-token or per-channel)."""
set_random_seed(seed)
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, scale_2d = scaled_quantize(
x, GroupShape(*group_shape), FP8_DTYPE, compute_dtype=torch.float32
)
# Flatten scale to 1D for testing 1D scale path
scale_1d = scale_2d.flatten()
ops_out, ops_scale = ops.scaled_fp8_quant(
x, scale=scale_1d, group_shape=group_shape
)
torch.testing.assert_close(scale_1d, ops_scale)
torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=0.12, atol=0.0)
opcheck_fp8_quant(ops_out, x, scale=scale_1d, group_shape=group_shape)
-29
View File
@@ -1,29 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from tests.kernels.utils import opcheck
from vllm import _custom_ops as ops # noqa: F401
def test_gptq_shuffle_opcheck():
weight = torch.randint(
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
)
bit = 4
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
def test_gptq_gemm_opcheck():
a = torch.rand((240, 4096), device="cuda", dtype=torch.float16)
weight = torch.randint(
-2000000, 2000000, (512, 6144), device="cuda", dtype=torch.int32
)
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
use_exllama = True
bit = 4
# Test both GPTQv1 and GPTQv2 format
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
import torch
from compressed_tensors.transform import deterministic_hadamard_matrix
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
if current_platform.is_rocm():
pytest.skip(
"These tests require hadacore_transform, not supported on ROCm.",
allow_module_level=True,
)
@pytest.mark.parametrize("batch_size", [1, 32])
@pytest.mark.parametrize("hidden_dim", [2**n for n in range(10)])
def test_hadacore(batch_size, hidden_dim, dtype=torch.bfloat16, device="cuda"):
x = torch.eye(hidden_dim, dtype=dtype, device=device)
hadamard = deterministic_hadamard_matrix(
hidden_dim, dtype=torch.float64, device="cuda"
) / math.sqrt(hidden_dim)
y = ops.hadacore_transform(x.clone())
y_true = (x.to(hadamard.dtype) @ hadamard.T).to(y.dtype)
assert torch.allclose(y, y_true)
y = ops.hadacore_transform(y)
assert torch.allclose(y, x)
File diff suppressed because it is too large Load Diff
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/sgl-project/sglang/blob/main/test/srt/test_int8_kernel.py
import itertools
import pytest
import torch
from tests.kernels.quant_utils import native_per_token_group_quant_int8
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import fused_experts
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_quant_int8,
)
from vllm.platforms import current_platform
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
pytest.skip(
"INT8 Triton kernels require a CUDA-alike or XPU device",
allow_module_level=True,
)
if current_platform.is_cuda_alike() and not current_platform.has_device_capability(
(7, 0)
):
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization"""
A = A.to(torch.float32)
B = B.to(torch.float32)
assert A.shape[-1] == B.shape[-1], "Dimension mismatch"
assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor"
# Reshape input
M = A.numel() // A.shape[-1]
B = B.t() # Transpose weight matrix
N, K = B.shape
origin_C_shape = A.shape[:-1] + (K,)
A = A.reshape(M, N)
# As is per-token [M, 1], Bs is per-column [1, K]
C = torch.matmul(A, B) # [M, K]
C = As * C * Bs.view(1, -1) # Broadcast per-column scale
return C.reshape(origin_C_shape).to(output_dtype)
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, topk, topk_weight, topk_ids):
"""This function performs fused moe with per-column int8 quantization
using native torch."""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = per_token_quant_int8(a)
# Repeat tokens to match topk
a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
# Also repeat the scale
a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1]
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
# Calculate routing
topk_weight = topk_weight.view(-1)
topk_ids = topk_ids.view(-1)
# Process each expert
for i in range(w1.shape[0]):
mask = topk_ids == i
if mask.sum():
# First MLP layer: note that a_s is now per-token
inter_out = native_w8a8_per_token_matmul(
a_q[mask], w1[i], a_s[mask], w1_s[i], output_dtype=a.dtype
)
# Activation function
act_out = SiluAndMul().forward_native(inter_out)
# Quantize activation output with per-token
act_out_q, act_out_s = per_token_quant_int8(act_out)
# Second MLP layer
out[mask] = native_w8a8_per_token_matmul(
act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype
)
# Apply routing weights and sum
return (
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
).sum(dim=1)
DTYPES = [torch.half, torch.bfloat16]
M = [1, 33]
N = [128, 1024]
K = [256, 4096]
E = [8]
TOP_KS = [2, 6]
SEEDS = [0]
@pytest.mark.parametrize(
"M, N, K, E, topk, dtype, seed",
itertools.product(M, N, K, E, TOP_KS, DTYPES, SEEDS),
)
@torch.inference_mode()
def test_w8a8_fp8_fused_moe(default_vllm_config, M, N, K, E, topk, dtype, seed):
torch.manual_seed(seed)
device = current_platform.device_type
# Initialize int8 quantization parameters
factor_for_scale = 1e-2
int8_max = 127
int8_min = -128
# Input tensor
# M * K
a = torch.randn((M, K), dtype=dtype, device=device) / 10
# Generate int8 weights
w1_fp32 = (
torch.rand(
(E, 2 * N, K),
dtype=torch.float32,
device=device,
)
- 0.5
) * 2
w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8)
w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32, device=device) - 0.5) * 2
w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8)
# Generate scale for each column (per-column quantization)
w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale
w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale
score = torch.randn((M, E), dtype=dtype, device=device)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weights, topk_ids = torch.topk(score, topk)
ref_out = torch_w8a8_per_column_moe(
a, w1, w2, w1_s, w2_s, topk, topk_weights, topk_ids
)
quant_config = FusedMoEQuantConfig.make(
torch.int8,
per_act_token_quant=True,
block_shape=None,
w1_scale=w1_s,
w2_scale=w2_s,
)
out = fused_experts(
a,
w1,
w2,
topk_weights,
topk_ids,
quant_config=quant_config,
)
# Check results
rel_diff = torch.mean(
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
assert rel_diff < 0.05
@pytest.mark.parametrize("num_tokens", [1, 33, 64])
@pytest.mark.parametrize("hidden_dim", [128, 1024])
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_per_token_quant_int8(num_tokens, hidden_dim, dtype):
"""Covers the round_int8 Triton kernel via per_token_quant_int8."""
torch.manual_seed(0)
device = current_platform.device_type
x = torch.randn((num_tokens, hidden_dim), dtype=dtype, device=device) * 8
q, s = per_token_quant_int8(x)
# Per-token int8 quant is per-token-group quant with group_size = hidden_dim.
ref_q, ref_s = native_per_token_group_quant_int8(x, hidden_dim)
assert q.dtype == torch.int8
assert q.shape == (num_tokens, hidden_dim)
assert s.shape == (num_tokens, 1)
assert torch.allclose(s.float(), ref_s.float(), atol=1e-3, rtol=1e-3)
# Quantized values may differ by at most 1 ULP due to rounding.
assert torch.all((q.float() - ref_q.float()).abs() <= 1)
@@ -1,450 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the machete kernel.
Run `pytest tests/kernels/quantization/test_machete_mm.py`.
"""
import math
from dataclasses import dataclass, fields
import pytest
import torch
from tests.kernels.utils import opcheck
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.machete_utils import (
query_machete_supported_group_sizes,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
pack_rows,
quantize_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
if current_platform.is_rocm():
pytest.skip(
"These tests require machete_prepack_B, not supported on ROCm.",
allow_module_level=True,
)
CUDA_DEVICES = [
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
]
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
# unit tests to a common utility function. Currently the use of
# `is_quant_method_supported` conflates kernels with quantization methods
# an assumption which is breaking down as quantizations methods can have
# have kernels and some kernels support multiple quantization methods.
capability = current_platform.get_device_capability()
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
MNK_SHAPES = [
(1, 128, 128),
(1, 8192, 28672),
(13, 8192, 4096),
(26, 4096, 8192),
(64, 4096, 4096),
(64, 8192, 28672),
(257, 128, 4096),
(257, 4224, 4160),
(1024, 8192, 4096),
]
@dataclass
class TypeConfig:
act_type: torch.dtype
weight_type: ScalarType
output_type: torch.dtype | None
group_scale_type: torch.dtype | None
group_zero_type: torch.dtype | None
channel_scale_type: torch.dtype | None
token_scale_type: torch.dtype | None
@dataclass
class Tensors:
w_ref: torch.Tensor
a_ref: torch.Tensor
a: torch.Tensor
w_q: torch.Tensor
w_g_s: torch.Tensor | None
w_g_zp: torch.Tensor | None
w_ch_s: torch.Tensor | None
w_tok_s: torch.Tensor | None
# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
# Ch Scales Type, Tok Scales Type)
# NOTE: None "Scale Type" means the act type is floating point
# None "Output Type" means the output type is the same as the act type
TestTypeTuple = tuple[
list[torch.dtype], ScalarType, torch.dtype | None, torch.dtype | None, bool
]
TEST_TYPES = [
# GPTQ style
*(
TypeConfig(
act_type=a_type,
weight_type=w_type,
output_type=None,
group_scale_type=a_type,
group_zero_type=None,
channel_scale_type=None,
token_scale_type=None,
)
for w_type in [scalar_types.uint4b8, scalar_types.uint8b128]
for a_type in [torch.float16, torch.bfloat16]
),
# AWQ style
*(
TypeConfig(
act_type=a_type,
weight_type=w_type,
output_type=None,
group_scale_type=a_type,
group_zero_type=a_type,
channel_scale_type=None,
token_scale_type=None,
)
for w_type in [scalar_types.uint4, scalar_types.uint8]
for a_type in [torch.float16, torch.bfloat16]
),
# # QQQ style
# *(TypeConfig(act_type=torch.int8,
# weight_type=scalar_types.uint4b8,
# output_type=torch.float16,
# group_scale_type=group_scale_type,
# group_zero_type=None,
# channel_scale_type=torch.float,
# token_scale_type=torch.float)
# for group_scale_type in [None, torch.float16]),
# *(TypeConfig(act_type=torch.float8_e4m3fn,
# weight_type=scalar_types.uint4b8,
# output_type=torch.float16,
# group_scale_type=group_scale_type,
# group_zero_type=None,
# channel_scale_type=torch.float,
# token_scale_type=torch.float)
# for group_scale_type in [None, torch.float16]),
]
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
# unit tests to a common utility function. Currently the use of
# `is_quant_method_supported` conflates kernels with quantization methods
# an assumption which is breaking down as quantizations methods can have
# have kernels and some kernels support multiple quantization methods.
IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
def rand_data(shape, dtype=torch.float16, scale=1, offset=0):
if dtype.is_floating_point:
return (scale * torch.rand(shape, device="cuda") - offset).to(dtype)
else:
return torch.randint(-8, 7, shape, dtype=dtype, device="cuda")
def maybe_convert_zeropoints(zps: torch.Tensor | None, s: torch.Tensor):
return zps if zps is None else -1 * s * (zps.to(s.dtype))
def group_size_valid(shape: tuple[int, int, int], group_size: int | None) -> bool:
return group_size is None or group_size == -1 or shape[2] % group_size == 0
def machete_quantize_and_pack(
atype: torch.dtype,
w: torch.Tensor,
wtype: ScalarType,
stype: torch.dtype | None,
group_size: int | None,
zero_points: bool = False,
):
assert wtype.is_integer(), "TODO: support floating point weights"
w_ref, w_q, w_s, w_zp = quantize_weights(
w,
wtype,
group_size=group_size,
zero_points=zero_points,
# to match how the kernel applies zps
ref_zero_points_after_scales=True,
)
w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
w_q = w_q.t().contiguous().t() # convert to col major
w_q_machete = ops.machete_prepack_B(w_q, atype, wtype, stype)
opcheck(torch.ops._C.machete_prepack_B, (w_q, atype, wtype.id, stype))
return w_ref, w_q_machete, w_s, w_zp
def create_test_tensors(
shape: tuple[int, int, int],
types: TypeConfig,
group_size: int | None,
subset_stride_factor: int | None = None,
) -> Tensors:
m, n, k = shape
factor = subset_stride_factor or 1
print(
"create_test_tensors, shape:", shape, "types:", types, "group_size:", group_size
)
a = rand_data((m * factor, k * factor), types.act_type, scale=3, offset=2)
w = rand_data((k * factor, n * factor), types.act_type, scale=3, offset=1)
if factor > 1:
a = a[0:m, 0:k]
w = w[0:k, 0:n]
if types.group_scale_type is not None:
w = w.to(types.group_scale_type)
if w.dtype.itemsize == 1:
w = w.to(torch.float16)
w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
a.dtype,
w,
types.weight_type,
types.group_scale_type,
group_size,
types.group_zero_type is not None,
)
if not a.dtype.is_floating_point:
aiinfo = torch.iinfo(a.dtype)
w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
a_ref = a.to(torch.float32)
w_ref = w_ref.to(torch.float32)
w_ch_s = (
None
if types.channel_scale_type is None
else rand_data((n,), types.channel_scale_type)
)
w_tok_s = (
None
if types.token_scale_type is None
else rand_data((m,), types.token_scale_type)
)
return Tensors(
w_ref=w_ref,
a_ref=a_ref,
a=a,
w_q=w_q_packed,
w_g_s=w_s,
w_g_zp=maybe_convert_zeropoints(w_zp, w_s),
w_ch_s=w_ch_s,
w_tok_s=w_tok_s,
)
# None stype means scales use the same dtype as a
def machete_mm_test_helper(
types: TypeConfig,
tensors: Tensors,
group_size: int | None = None,
schedule: str | None = None,
):
output_ref = torch.matmul(tensors.a_ref, tensors.w_ref)
output_ref_type = output_ref.dtype
if tensors.w_ch_s is not None:
output_ref = (
output_ref.to(tensors.w_ch_s.dtype) * tensors.w_ch_s.unsqueeze(0)
).to(output_ref_type)
if tensors.w_tok_s is not None:
output_ref = (
output_ref.to(tensors.w_tok_s.dtype) * tensors.w_tok_s.unsqueeze(1)
).to(output_ref_type)
output = ops.machete_mm(
a=tensors.a,
b_q=tensors.w_q,
b_type=types.weight_type,
b_group_scales=tensors.w_g_s,
b_group_zeros=tensors.w_g_zp,
b_group_size=group_size,
b_channel_scales=tensors.w_ch_s,
a_token_scales=tensors.w_tok_s,
out_type=types.output_type,
schedule=schedule,
)
print(output)
print(output_ref)
# Relax atol as our reduction dim becomes larger (more rounding error)
# Relax atol when we have zeropoints since the way machete applies
# zeropoints (after scales) causes noise around 0
atol = (
1
if tensors.w_g_zp is not None
else min(5e-2 * math.sqrt(tensors.a.shape[1]), 1)
)
rtol = 1e-1 if tensors.a.element_size() >= 2 else 2e-1
torch.testing.assert_close(
output, output_ref.to(output.dtype), rtol=rtol, atol=atol
)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
)
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
@pytest.mark.parametrize("types", TEST_TYPES)
def test_machete_all_schedules(shape, types: TypeConfig):
group_sizes: list[int | None] = []
if types.group_scale_type is None:
group_sizes = [None]
else:
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
for group_size in group_sizes:
if not group_size_valid(shape, group_size):
continue
tensors = create_test_tensors(shape, types, group_size)
print(f"MNK = {shape}")
for schedule in ops.machete_supported_schedules(
types.act_type,
types.weight_type,
group_scales_type=types.group_scale_type,
group_zeros_type=types.group_scale_type,
out_type=types.output_type,
):
print(f"Testing schedule {schedule}")
machete_mm_test_helper(types, tensors, group_size, schedule)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
)
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
@pytest.mark.parametrize("types", TEST_TYPES)
def test_machete_heuristic(shape, types: TypeConfig):
group_sizes: list[int | None] = []
if types.group_scale_type is None:
group_sizes = [None]
else:
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
for group_size in group_sizes:
if not group_size_valid(shape, group_size):
continue
tensors = create_test_tensors(shape, types, group_size)
machete_mm_test_helper(types, tensors, group_size)
# Test working on other devices
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
)
@pytest.mark.parametrize("device", CUDA_DEVICES)
def test_machete_devices(device: str):
group_size = 128
type_config = TypeConfig(
act_type=torch.float16,
weight_type=scalar_types.uint4b8,
output_type=None,
group_scale_type=torch.float16,
group_zero_type=None,
channel_scale_type=None,
token_scale_type=None,
)
tensors = create_test_tensors((512, 4096, 4096), type_config, group_size)
for field in fields(Tensors):
tensor = getattr(tensors, field.name)
if isinstance(tensor, torch.Tensor):
setattr(tensors, field.name, tensor.to(device))
machete_mm_test_helper(type_config, tensors, group_size)
# Test working with a subset of A and B
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
)
def test_machete_subset():
group_size = 128
type_config = TypeConfig(
act_type=torch.float16,
weight_type=scalar_types.uint4b8,
output_type=None,
group_scale_type=torch.float16,
group_zero_type=None,
channel_scale_type=None,
token_scale_type=None,
)
tensors = create_test_tensors(
(512, 4096, 4096), type_config, group_size, subset_stride_factor=2
)
machete_mm_test_helper(type_config, tensors, group_size)
# Test to make sure cuda graphs work
class MacheteLayer(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
self.kwargs = kwargs
def forward(self, a):
return ops.machete_mm(a=a, **self.kwargs)
@pytest.mark.skipif(
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
)
def test_machete_cuda_graph():
m, n, k = 512, 4096, 4096
a = rand_data((m, k), torch.float16)
b = rand_data((k, n), torch.float16)
wtype = scalar_types.uint4b8
stype = torch.float16
group_size = 128
zero_points = False
w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
a.dtype, b, wtype, stype, group_size, zero_points
)
# Construct a trivial model with a single layer that calls a machete kernel
model = MacheteLayer(
b_q=w_q_packed,
b_type=wtype,
b_group_scales=w_s,
b_group_zeros=maybe_convert_zeropoints(w_zp, w_s),
b_group_size=group_size,
)
output_ref = torch.matmul(a, w_ref)
# Run the model with a cuda graph
stream = torch.cuda.Stream()
with torch.cuda.stream(stream):
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
output = model(a)
output.zero_()
g.replay()
# Relax atol as our reduction dim becomes larger (more rounding error)
# Relax atol when we have zeropoints since the way machete applies
# zeropoints (after scales) causes noise around 0
atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
@@ -1,622 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the marlin kernel.
Run `pytest tests/kernels/quantization/test_marlin_gemm.py`.
"""
import itertools
import pytest
import torch
from tests.kernels.utils import opcheck
from tests.quantization.utils import is_quant_method_supported
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.int8_utils import (
per_token_quant_int8,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
get_marlin_workspace,
marlin_make_empty,
marlin_make_workspace_new,
marlin_permute_bias,
query_marlin_supported_quant_types,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
rand_marlin_weight_mxfp4_like,
rand_marlin_weight_nvfp4_like,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
marlin_quant_fp8_torch,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
awq_marlin_quantize,
get_weight_perm,
marlin_quantize,
marlin_weights,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
awq_pack,
gptq_pack,
gptq_quantize_weights,
quantize_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.v1.worker.workspace import current_workspace_manager
if current_platform.is_rocm():
pytest.skip(
"These tests require marlin, which is not supported on ROCm.",
allow_module_level=True,
)
USE_ATOMIC_ADD_OPTS = [False, True]
USE_FP32_REDUCE_OPTS = [True]
MARLIN_K_CHUNKS = [128]
MARLIN_N_CHUNKS = [64, 256]
MARLIN_REPACK_NK_FACTORS = [
(4, 8),
(7, 5),
(13, 11),
]
MNK_FACTORS = [
(1, 1, 1),
(1, 4, 8),
(26, 37, 13),
(257, 13, 11),
]
DTYPES = [torch.float16, torch.bfloat16]
DENSE_MARLIN_QUANT_TEST_CONFIGS = [
# AWQ-INT4
{"b_type": scalar_types.uint4, "group_blocks": [-1, 2, 4, 8]},
# GPTQ-INT4
{
"b_type": scalar_types.uint4b8,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT8
{
"b_type": scalar_types.uint8b128,
"group_blocks": [-1, 2, 4, 8],
},
# FP8
{"b_type": scalar_types.float8_e4m3fn, "group_blocks": [-1, 8]},
# NVFP4
{"b_type": scalar_types.float4_e2m1f, "group_blocks": [1]},
# MXFP4
{
"a_type": [scalar_types.bfloat16],
"b_type": scalar_types.float4_e2m1f,
"group_blocks": [2],
},
# AWQ-INT4 with INT8 activation
{
"a_type": [scalar_types.int8],
"b_type": scalar_types.uint4,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT4 with INT8 activation
{
"a_type": [scalar_types.int8],
"b_type": scalar_types.uint4b8,
"group_blocks": [-1, 2, 4, 8],
},
# GPTQ-INT4 with FP8 activation
{
"a_type": [scalar_types.float8_e4m3fn],
"b_type": scalar_types.uint4b8,
"group_blocks": [-1, 2, 4, 8],
},
# AWQ-INT4 with FP8 activation
{
"a_type": [scalar_types.float8_e4m3fn],
"b_type": scalar_types.uint4,
"group_blocks": [-1, 2, 4, 8],
},
# MXFP4 with FP8 activation
{
"a_type": [scalar_types.float8_e4m3fn],
"b_type": scalar_types.float4_e2m1f,
"c_type": [scalar_types.bfloat16],
"group_blocks": [2],
},
]
def compute_max_diff(output, output_ref):
return torch.mean(torch.abs(output - output_ref)) / torch.mean(
torch.abs(output_ref)
)
def rand_data(shape, dtype=torch.float16):
return torch.randn(shape, dtype=dtype, device="cuda")
@pytest.mark.skipif(
not is_quant_method_supported("gptq_marlin"),
reason="Marlin is not supported on this GPU type.",
)
def test_marlin_int4_fp8_preprocess_without_zp():
qweight_unpacked = torch.randint(
0, 16, size=(2048, 2048), dtype=torch.int32, device="cuda"
)
qweight_packed = qweight_unpacked[:, ::2] * 16 + qweight_unpacked[:, 1::2]
qweight_packed = qweight_packed.to(torch.int8).view(torch.int32)
cuda_res = ops.marlin_int4_fp8_preprocess(qweight_packed)
torch_res = torch.where(
qweight_unpacked >= 8, qweight_unpacked - 8, 15 - qweight_unpacked
)
torch_res = torch_res[:, ::2] * 16 + torch_res[:, 1::2]
torch_res = torch_res.to(torch.int8).view(torch.int32)
assert (cuda_res == torch_res).all()
@pytest.mark.skipif(
not is_quant_method_supported("gptq_marlin"),
reason="Marlin is not supported on this GPU type.",
)
def test_marlin_int4_fp8_preprocess_awq():
group_size = 128
qweight_unpacked = torch.randint(
0, 16, size=(2048, 2048), dtype=torch.int32, device="cuda"
)
qzeros_unpacked = torch.randint(
0, 16, size=(2048 // group_size, 2048), dtype=torch.int32, device="cuda"
)
qweight_packed = qweight_unpacked[:, ::2] * 16 + qweight_unpacked[:, 1::2]
qweight_packed = qweight_packed.to(torch.int8).view(torch.int32)
qzeros_packed = qzeros_unpacked[:, ::2] * 16 + qzeros_unpacked[:, 1::2]
qzeros_packed = qzeros_packed.to(torch.int8).view(torch.int32)
cuda_res = ops.marlin_int4_fp8_preprocess(qweight_packed, qzeros_packed)
repeated_zp = qzeros_unpacked.repeat_interleave(group_size, 0)
torch_res = qweight_unpacked - repeated_zp
torch_res[torch_res < 0] = 15 - qweight_unpacked[torch_res < 0]
torch_res = torch_res[:, ::2] * 16 + torch_res[:, 1::2]
torch_res = torch_res.to(torch.int8).view(torch.int32)
assert (cuda_res == torch_res).all()
@pytest.mark.skipif(
not is_quant_method_supported("gptq_marlin"),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
@pytest.mark.parametrize("is_a_8bit", [True, False])
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
n_factor, k_factor = nk_factors
size_k = k_chunk * k_factor
size_n = n_chunk * n_factor
group_size = 128
# Normalize group_size
if group_size == -1:
group_size = size_k
assert group_size <= size_k
# Create input
b_weight = rand_data((size_k, size_n))
# Quantize
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
# Pack to GPTQ format
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
# Pack to Marlin format
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w_1 = marlin_weights(
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit
)
opcheck(
torch.ops._C.gptq_marlin_repack,
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
)
# Run Marlin repack GPU kernel
marlin_q_w_2 = ops.gptq_marlin_repack(
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.accelerator.synchronize()
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
@pytest.mark.skipif(
not is_quant_method_supported("gptq_marlin"),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(True))
@pytest.mark.parametrize("is_a_8bit", [True, False])
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
n_factor, k_factor = nk_factors
size_k = k_chunk * k_factor
size_n = n_chunk * n_factor
group_size = 128
# Create input
b_weight = rand_data((size_k, size_n))
# Quantize
w_ref, q_w, s, zp = quantize_weights(
b_weight, quant_type, group_size, zero_points=True
)
# Pack to AWQ format
q_w_awq = awq_pack(q_w, quant_type.size_bits, size_k, size_n)
# Pack to Marlin format
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
marlin_q_w_1 = marlin_weights(
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit
)
opcheck(
torch.ops._C.awq_marlin_repack,
(q_w_awq, size_k, size_n, quant_type.size_bits, is_a_8bit),
)
# Run Marlin repack GPU kernel
marlin_q_w_2 = ops.awq_marlin_repack(
q_w_awq, size_k, size_n, quant_type.size_bits, is_a_8bit
)
torch.accelerator.synchronize()
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
def marlin_generate_valid_test_cases():
all_combinations = itertools.product(
DENSE_MARLIN_QUANT_TEST_CONFIGS,
MNK_FACTORS,
MARLIN_N_CHUNKS,
MARLIN_K_CHUNKS,
USE_ATOMIC_ADD_OPTS,
USE_FP32_REDUCE_OPTS,
)
def is_invalid(
a_type,
b_type,
c_type,
group_blocks,
size_m,
size_n,
size_k,
use_atomic_add,
use_fp32_reduce,
):
if use_atomic_add:
if use_fp32_reduce:
return False
if (
c_type == scalar_types.bfloat16
and torch.cuda.get_device_capability()[0] < 9
):
return False
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
if group_size > 0 and size_k % group_size != 0:
return False
if group_size == size_k:
return False
return a_type.size_bits < 16 or a_type is c_type
cases = []
for case in all_combinations:
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
size_m = mnk_factors[0]
size_n = mnk_factors[1] * n_chunk
size_k = mnk_factors[2] * k_chunk
f16_types = [scalar_types.float16, scalar_types.bfloat16]
inner_combinations = itertools.product(
quant_test_config.get("a_type", f16_types),
[quant_test_config["b_type"]],
quant_test_config.get("c_type", f16_types),
quant_test_config["group_blocks"],
) # type: ignore[call-overload]
for sub_case in inner_combinations:
if (
sub_case[0] == scalar_types.float8_e4m3fn
and not current_platform.is_device_capability(89)
and not current_platform.is_device_capability_family(120)
):
continue
args = sub_case + (size_m, size_n, size_k) + case[4:]
if is_invalid(*args):
cases.append(args)
return cases
@pytest.mark.skipif(
not is_quant_method_supported("gptq_marlin"),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize(
(
"a_type, b_type, c_type, group_blocks,"
"size_m, size_n, size_k,"
"use_atomic_add, use_fp32_reduce"
),
marlin_generate_valid_test_cases(),
)
def test_marlin_gemm(
a_type,
b_type,
c_type,
group_blocks,
size_m,
size_n,
size_k,
use_atomic_add,
use_fp32_reduce,
):
has_zp = b_type in [scalar_types.uint4, scalar_types.uint8]
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
if c_type == scalar_types.float16:
dtype = torch.float16
elif c_type == scalar_types.bfloat16:
dtype = torch.bfloat16
else:
raise RuntimeError("unsupported c_type")
if a_type == scalar_types.int8:
a_dtype = torch.int8
elif a_type == scalar_types.float8_e4m3fn:
a_dtype = torch.float8_e4m3fn
else:
a_dtype = dtype
a_input = rand_data((size_m, size_k), dtype=dtype)
b_weight = rand_data((size_k, size_n), dtype=dtype)
if b_type == scalar_types.float4_e2m1f:
if group_size == 16:
w_ref, marlin_q_w, marlin_s, marlin_s2 = rand_marlin_weight_nvfp4_like(
b_weight.T, group_size, input_dtype=a_dtype
)
else:
w_ref, marlin_q_w, marlin_s = rand_marlin_weight_mxfp4_like(
b_weight.T, group_size, input_dtype=a_dtype
)
marlin_s2 = None
marlin_zp = None
elif b_type == scalar_types.float8_e4m3fn:
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
b_weight.T, group_size, input_dtype=a_dtype
)
marlin_zp = None
marlin_s2 = None
elif has_zp:
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
marlin_s2 = None
else:
w_ref, marlin_q_w, marlin_s = marlin_quantize(
b_weight, b_type, group_size, input_dtype=a_dtype
)
marlin_zp = None
marlin_s2 = None
workspace = marlin_make_workspace_new(w_ref.device)
if a_type == scalar_types.int8:
a_input, a_scales = per_token_quant_int8(a_input)
a_input_ref = a_input.to(a_scales.dtype) * a_scales.view(-1, 1)
a_input_ref = a_input_ref.to(dtype)
if group_size != -1:
a_scales = a_scales / 4096 * marlin_s.max()
a_scales = a_scales.float()
marlin_s = marlin_s / marlin_s.max() * 4096
marlin_s = marlin_s.round().to(torch.int16).view(dtype)
elif a_type == scalar_types.float8_e4m3fn:
a_input, a_scales = ops.scaled_fp8_quant(a_input, use_per_token_if_dynamic=True)
a_input_ref = a_input.to(a_scales.dtype) * a_scales.view(-1, 1)
a_input_ref = a_input_ref.to(dtype)
else:
assert a_type.size_bits == 16
a_input_ref = a_input
a_scales = None
output = torch.empty((size_m, size_n), dtype=dtype, device=a_input.device)
output = ops.marlin_gemm(
a_input,
output,
marlin_q_w,
None,
marlin_s,
a_scales,
marlin_s2,
marlin_zp,
workspace,
b_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
use_atomic_add=use_atomic_add,
use_fp32_reduce=use_fp32_reduce,
is_zp_float=False,
)
output_ref = torch.matmul(a_input_ref, w_ref)
max_diff = compute_max_diff(output, output_ref)
assert max_diff < 0.04
def test_marlin_persistent_locks_cuda_graph_streams(workspace_init):
"""Compiled GEMMs select stream-local locks at runtime and survive replay."""
size_m, size_k, size_n = 32, 1024, 2048
inputs = [rand_data((size_m, size_k)) for _ in range(2)]
weight = rand_data((size_k, size_n))
quant_type = scalar_types.uint4b8
w_ref, qweight, scales = marlin_quantize(weight, quant_type, 128)
zeros = marlin_make_empty(scales.device)
streams = [torch.cuda.Stream() for _ in inputs]
graphs = [torch.cuda.CUDAGraph() for _ in inputs]
locks = []
outputs = []
@torch.compile(backend="eager", fullgraph=True)
def run(x):
return torch.ops.vllm.marlin_gemm(
x,
qweight,
None,
scales,
None,
None,
zeros,
None,
quant_type.id,
size_m,
size_n,
size_k,
use_atomic_add=False,
use_fp32_reduce=True,
)
for stream, graph, x in zip(streams, graphs, inputs):
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
locks.append(get_marlin_workspace(x.device))
run(x)
with torch.cuda.graph(graph, stream=stream):
outputs.append(run(x))
torch.cuda.current_stream().wait_stream(stream)
assert locks[0].data_ptr() != locks[1].data_ptr()
current_workspace_manager().lock()
for _ in range(3):
for stream, graph, x, lock in zip(streams, graphs, inputs, locks):
with torch.cuda.stream(stream):
assert get_marlin_workspace(x.device) is lock
x.add_(0.1)
graph.replay()
for stream in streams:
torch.cuda.current_stream().wait_stream(stream)
for x, output, lock in zip(inputs, outputs, locks):
assert compute_max_diff(output, x @ w_ref) < 0.04
assert torch.count_nonzero(lock) == 0
def test_marlin_gemm_subset_input():
quant_type = scalar_types.uint4b8
group_size = 128
size_m, size_k, size_n = 32, 1024, 2048
big_m = size_m * 2
big_k = size_k * 2
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
b_weight = rand_data((size_k, size_n))
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
a_input,
None,
marlin_q_w,
None,
marlin_s,
None,
None,
marlin_zp,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
)
output_ref = torch.matmul(a_input, w_ref)
torch.accelerator.synchronize()
max_diff = compute_max_diff(output, output_ref)
assert max_diff < 0.04
@pytest.mark.parametrize("size_m", [1, 256])
def test_marlin_gemm_with_bias(size_m):
quant_type = scalar_types.uint4b8
group_size = 128
size_k, size_n = 1024, 2048
a_input = rand_data((size_m, size_k))
b_weight = rand_data((size_k, size_n))
b_bias = rand_data((size_n,)) * 10
marlin_bias = marlin_permute_bias(b_bias)
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
marlin_zp = marlin_make_empty(marlin_s.device)
workspace = marlin_make_workspace_new(a_input.device)
output = ops.marlin_gemm(
a_input,
None,
marlin_q_w,
marlin_bias,
marlin_s,
None,
None,
marlin_zp,
workspace,
quant_type,
a_input.shape[0],
b_weight.shape[1],
a_input.shape[1],
use_atomic_add=False,
use_fp32_reduce=True,
is_zp_float=False,
)
output_ref = torch.matmul(a_input, w_ref) + b_bias.view(1, -1)
torch.accelerator.synchronize()
max_diff = compute_max_diff(output, output_ref)
assert max_diff < 0.04
@@ -1,851 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Marlin thread-tile padding of TP-sharded weight shapes.
Run `pytest tests/kernels/quantization/test_marlin_tile_padding.py`.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
GPTQ_MARLIN_TILE,
apply_gptq_marlin_linear,
get_marlin_workspace,
marlin_make_empty,
marlin_make_workspace_new,
marlin_moe_padded_intermediate,
marlin_pad_qweight,
marlin_pad_scales,
marlin_padded_nk,
marlin_permute_scales,
marlin_repacked_nk,
marlin_zero_points,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
apply_fp4_marlin_linear,
is_fp4_marlin_supported,
prepare_fp4_layer_for_marlin,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
apply_fp8_marlin_linear,
apply_mxfp8_marlin_linear,
is_fp8_marlin_supported,
prepare_fp8_layer_for_marlin,
prepare_mxfp8_layer_for_marlin,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
gptq_pack,
gptq_quantize_weights,
quantize_weights,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
# (size_n, size_k) rank-local shapes that violate Marlin tile alignment,
# e.g. produced by TP-sharding dims that are valid at TP=1.
ODD_SHAPES = [
(200, 288), # N padded
(256, 208), # K padded
(200, 208), # both padded
(4640, 512), # Nemotron-Super-120B q_proj shard at TP=4
]
ALIGNED_SHAPES = [(64, 128), (128, 64), (256, 256), (4608, 4096)]
def _is_tile_aligned(size_n: int, size_k: int) -> bool:
return (size_n % 64 == 0 and size_k % 128 == 0) or (
size_n % 128 == 0 and size_k % 64 == 0
)
@pytest.mark.parametrize("shape", ODD_SHAPES + ALIGNED_SHAPES)
@pytest.mark.parametrize("group_size", [-1, 16, 32, 64, 128])
def test_marlin_padded_nk(shape, group_size):
size_n, size_k = shape
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
assert padded_n >= size_n and padded_k >= size_k
assert _is_tile_aligned(padded_n, padded_k)
if group_size > 0:
assert padded_k % group_size == 0
# Aligned shapes must pass through unchanged (zero hot-path cost).
if _is_tile_aligned(size_n, size_k) and (
group_size <= 0 or size_k % group_size == 0
):
assert (padded_n, padded_k) == (size_n, size_k)
# Minimal: no valid shape with a smaller padded area exists.
area = padded_n * padded_k
for cand_n in range(size_n, padded_n + 1):
for cand_k in range(size_k, padded_k + 1):
if (
_is_tile_aligned(cand_n, cand_k)
and (group_size <= 0 or cand_k % group_size == 0)
and cand_n * cand_k < area
):
pytest.fail(f"({cand_n}, {cand_k}) beats ({padded_n}, {padded_k})")
# Apply-time derivation from the repacked-tensor shape must round-trip.
for num_bits in (4, 8):
pack_factor = 32 // num_bits
repacked_shape = (
padded_k // GPTQ_MARLIN_TILE,
padded_n * GPTQ_MARLIN_TILE // pack_factor,
)
repacked = torch.empty(repacked_shape, device="meta")
assert marlin_repacked_nk(repacked, num_bits) == (padded_n, padded_k)
def test_marlin_pad_helpers_shapes():
size_n, size_k, group_size = 200, 208, 16
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
qweight = torch.zeros(size_k // 8, size_n, dtype=torch.int32)
padded = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
assert padded.shape == (padded_k // 8, padded_n)
scales = torch.ones(size_k // group_size, size_n)
padded = marlin_pad_scales(scales, size_n, size_k, padded_n, padded_k, group_size)
assert padded.shape == (padded_k // group_size, padded_n)
assert padded[:, size_n:].abs().sum() == 0
channelwise = torch.ones(1, size_n)
padded = marlin_pad_scales(channelwise, size_n, size_k, padded_n, padded_k, -1)
assert padded.shape == (1, padded_n)
# Rank-local MoE intermediate sizes. group<=0 / 32 with a non-multiple-of-64
# size is where tile padding triggers; 64/128 are already tile-aligned.
MOE_INTERMEDIATE_SIZES = [64, 96, 100, 176, 192, 256, 2816]
@pytest.mark.parametrize("intermediate", MOE_INTERMEDIATE_SIZES)
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
def test_marlin_moe_padded_intermediate(intermediate, group_size):
# The MoE gate only admits shapes where the group does not straddle the
# boundary, i.e. group divides the intermediate size.
if group_size > 0 and intermediate % group_size != 0:
pytest.skip("group straddles the boundary; rejected by the MoE gate")
padded = marlin_moe_padded_intermediate(intermediate, group_size)
assert padded >= intermediate
# Valid MoE thread tile: gate-up n = 2*intermediate % 128, down k % 64.
assert (2 * padded) % 128 == 0
assert padded % 64 == 0
if group_size > 0:
assert padded % group_size == 0
# Minimal: no smaller valid intermediate exists.
for cand in range(intermediate, padded):
if (
(2 * cand) % 128 == 0
and cand % 64 == 0
and (group_size <= 0 or cand % group_size == 0)
):
pytest.fail(f"{cand} beats {padded}")
# Already-tile-aligned sizes pass through unchanged (zero hot-path cost).
if intermediate % 64 == 0:
assert padded == intermediate
def test_marlin_moe_pad_helpers_shapes():
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
_pad_rows,
_pad_w13_bias,
_pad_w13_shard_cols,
)
E, rows, N, padded_N = 2, 8, 96, 128
# w13 stores the two gate/up shards along the last dim; padding each shard
# must preserve the loaded values and zero the padded columns.
w13 = torch.arange(E * rows * 2 * N).reshape(E, rows, 2 * N).float()
padded = _pad_w13_shard_cols(w13, N, padded_N)
assert padded.shape == (E, rows, 2 * padded_N)
shards = padded.view(E, rows, 2, padded_N)
orig = w13.view(E, rows, 2, N)
assert torch.equal(shards[..., :N], orig)
assert shards[..., N:].abs().sum() == 0
# w2 stores the intermediate dim in the rows.
w2 = torch.ones(E, N // 32, 16)
padded = _pad_rows(w2, padded_N // 32)
assert padded.shape == (E, padded_N // 32, 16)
assert padded[:, N // 32 :, :].abs().sum() == 0
bias = torch.arange(E * 2 * N).reshape(E, 2 * N).float()
padded = _pad_w13_bias(bias, N, padded_N)
assert padded.shape == (E, 2 * padded_N)
bias_shards = padded.view(E, 2, padded_N)
assert torch.equal(bias_shards[..., :N], bias.view(E, 2, N))
assert bias_shards[..., N:].abs().sum() == 0
def _gpu_marlin_unsupported() -> bool:
return not (
current_platform.is_cuda() and current_platform.has_device_capability(80)
)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", ODD_SHAPES)
@pytest.mark.parametrize("use_bias", [False, True])
def test_fp8_marlin_padded_round_trip(shape, use_bias):
size_n, size_k = shape
dtype = torch.float16
layer = torch.nn.Module()
layer.output_size_per_partition = size_n
layer.input_size_per_partition = size_k
layer.orig_dtype = dtype
weight = torch.randn(size_k, size_n, dtype=dtype, device="cuda") / size_k**0.5
scale = weight.abs().max() / 448
weight_fp8 = (weight / scale).to(torch.float8_e4m3fn)
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(
scale.to(torch.float32), requires_grad=False
)
bias = None
if use_bias:
bias = torch.randn(size_n, dtype=dtype, device="cuda")
layer.bias = torch.nn.Parameter(bias.clone(), requires_grad=False)
prepare_fp8_layer_for_marlin(layer, size_k_first=True)
x = torch.randn(8, size_k, dtype=dtype, device="cuda")
output = apply_fp8_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
workspace=get_marlin_workspace(layer.weight.device),
size_n=size_n,
size_k=size_k,
bias=layer.bias if use_bias else None,
)
ref = x @ (weight_fp8.to(dtype) * scale.to(dtype))
if use_bias:
ref = ref + bias
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
def _dequant_fp4(packed: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
"""Dequantize packed e2m1 nibbles (N, K // 2) -> (N, K) in dtype."""
lo = (packed & 0b10000000) | ((packed & 0b01110000) >> 2)
lo = lo.view(torch.float8_e4m3fn).to(dtype) * (2**6)
hi_bits = packed << 4
hi = (hi_bits & 0b10000000) | ((hi_bits & 0b01110000) >> 2)
hi = hi.view(torch.float8_e4m3fn).to(dtype) * (2**6)
return torch.cat([hi.unsqueeze(2), lo.unsqueeze(2)], 2).view(packed.size(0), -1)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp4_marlin_supported(),
reason="FP4 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", ODD_SHAPES)
def test_nvfp4_marlin_padded_round_trip(shape):
size_n, size_k = shape
group_size = 16
dtype = torch.float16
layer = torch.nn.Module()
layer.output_size_per_partition = size_n
layer.input_size_per_partition = size_k
layer.params_dtype = dtype
packed = torch.randint(
0, 256, (size_n, size_k // 2), dtype=torch.uint8, device="cuda"
)
scales = (torch.rand(size_n, size_k // group_size, device="cuda") + 0.25).to(
torch.float8_e4m3fn
)
global_scale = torch.tensor([0.002], dtype=torch.float32, device="cuda")
ref_weight = (
_dequant_fp4(packed, dtype)
* scales.to(dtype).repeat_interleave(group_size, 1)
* global_scale.to(dtype)
)
layer.weight = torch.nn.Parameter(packed, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
layer.weight_global_scale = torch.nn.Parameter(global_scale, requires_grad=False)
prepare_fp4_layer_for_marlin(layer)
x = torch.randn(8, size_k, dtype=dtype, device="cuda") / size_k**0.5
output = apply_fp4_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
weight_global_scale=layer.weight_global_scale,
workspace=get_marlin_workspace(layer.weight.device),
size_n=size_n,
size_k=size_k,
)
ref = x @ ref_weight.T
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(
_gpu_marlin_unsupported(),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", ODD_SHAPES)
@pytest.mark.parametrize("group_size", [-1, 128])
def test_gptq_marlin_padded_round_trip(shape, group_size):
"""Pad-then-repack a GPTQ int4 weight the way MarlinLinearKernel does and
check the GEMM against the dequantized reference.
Symmetric int4's quantized zero decodes to -8, so this exercises the
zero-padded-scales cancellation, not just zero weights.
"""
size_n, size_k = shape
if group_size > 0 and size_k % group_size != 0:
pytest.skip("group must divide the rank-local K (not fixable by padding)")
dtype = torch.float16
quant_type = scalar_types.uint4b8
device = torch.device("cuda")
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
)
s = marlin_pad_scales(s, size_n, size_k, padded_n, padded_k, group_size)
marlin_s = marlin_permute_scales(
s, size_k=padded_k, size_n=padded_n, group_size=group_size
)
x = torch.randn(8, size_k, dtype=dtype, device=device)
output = apply_gptq_marlin_linear(
input=x,
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_make_empty(device),
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
)
ref = x @ w_ref
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
def test_fp8_block_marlin_padded_round_trip(shape):
"""Block-quantized FP8 (e.g. Nemotron NVFP4 checkpoints' FP8 layers):
group_size=128 exercises the lcm K-alignment in marlin_padded_nk and the
weight_scale_inv group-wise scale padding."""
size_n, size_k = shape
block = 128
dtype = torch.float16
layer = torch.nn.Module()
layer.output_size_per_partition = size_n
layer.input_size_per_partition = size_k
layer.orig_dtype = dtype
layer.weight_block_size = [block, block]
weight = torch.randn(size_n, size_k, dtype=dtype, device="cuda") / size_k**0.5
n_blocks, k_blocks = (size_n + block - 1) // block, size_k // block
padded = torch.zeros(n_blocks * block, size_k, dtype=dtype, device="cuda")
padded[:size_n] = weight
scales = padded.view(n_blocks, block, k_blocks, block).abs().amax(dim=(1, 3)) / 448
scales_expanded = scales.repeat_interleave(block, 0)[:size_n].repeat_interleave(
block, 1
)
weight_fp8 = (weight / scales_expanded).to(torch.float8_e4m3fn)
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
layer.weight_scale_inv = torch.nn.Parameter(
scales.to(torch.float32), requires_grad=False
)
prepare_fp8_layer_for_marlin(layer, size_k_first=False)
x = torch.randn(8, size_k, dtype=dtype, device="cuda")
output = apply_fp8_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale_inv,
workspace=get_marlin_workspace(layer.weight.device),
size_n=size_n,
size_k=size_k,
bias=None,
)
ref = x @ (weight_fp8.to(dtype) * scales_expanded.to(dtype)).T
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize(
"shape", [(32, 256), (96, 256), (130, 256), (200, 288), (4640, 512)]
)
def test_mxfp8_marlin_padded_round_trip(shape):
"""MXFP8 exercises the e8m0 scale path, where padded 0.0 scales clamp to
2^-127 instead of zero and must still contribute nothing."""
size_n, size_k = shape
group_size = 32
# The e8m0-scale Marlin kernels are only instantiated for bf16 activations.
dtype = torch.bfloat16
layer = torch.nn.Module()
layer.output_size_per_partition = size_n
layer.input_size_per_partition = size_k
weight_fp8 = (torch.randn(size_n, size_k, dtype=dtype, device="cuda") / 4).to(
torch.float8_e4m3fn
)
# e8m0 exponents around 1.0 (127): scales in [2^-6, 2^0]
scales = torch.randint(
121, 128, (size_n, size_k // group_size), dtype=torch.uint8, device="cuda"
)
ref_weight = weight_fp8.to(dtype) * (
2.0 ** (scales.to(dtype) - 127)
).repeat_interleave(group_size, 1)
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
prepare_mxfp8_layer_for_marlin(layer)
x = torch.randn(8, size_k, dtype=dtype, device="cuda") / size_k**0.5
output = apply_mxfp8_marlin_linear(
input=x,
weight=layer.weight,
weight_scale=layer.weight_scale,
workspace=get_marlin_workspace(layer.weight.device),
size_n=size_n,
size_k=size_k,
)
ref = x @ ref_weight.T
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(
_gpu_marlin_unsupported(),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
def test_awq_zp_marlin_padded_round_trip(shape):
"""AWQ-style uint4 with runtime zero-points, padded the way
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0."""
size_n, size_k = shape
group_size = 128
dtype = torch.float16
quant_type = scalar_types.uint4
device = torch.device("cuda")
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
w_ref, q_w, s, zp = quantize_weights(
weight, quant_type, group_size, zero_points=True
)
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
marlin_qweight = ops.gptq_marlin_repack(
b_q_weight=qweight,
size_k=padded_k,
size_n=padded_n,
num_bits=quant_type.size_bits,
)
s = marlin_pad_scales(s, size_n, size_k, padded_n, padded_k, group_size)
marlin_s = marlin_permute_scales(
s, size_k=padded_k, size_n=padded_n, group_size=group_size
)
zp = marlin_pad_scales(zp, size_n, size_k, padded_n, padded_k, group_size)
marlin_zp = marlin_zero_points(
zp,
size_k=padded_k // group_size,
size_n=padded_n,
num_bits=quant_type.size_bits,
)
x = torch.randn(8, size_k, dtype=dtype, device=device)
output = apply_gptq_marlin_linear(
input=x,
weight=marlin_qweight,
weight_scale=marlin_s,
weight_zp=marlin_zp,
workspace=marlin_make_workspace_new(device),
wtype=quant_type,
output_size_per_partition=size_n,
input_size_per_partition=size_k,
)
ref = x @ w_ref
assert output.shape == (8, size_n)
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
class _FakeLinear:
def __init__(self, size_n, size_k, input_size=None):
self.output_size_per_partition = size_n
self.input_size_per_partition = size_k
self.output_size = size_n
self.input_size = input_size if input_size is not None else size_k
def test_check_marlin_supports_layer_allow_tile_padding():
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_marlin_supports_layer,
)
# Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding
layer: LinearBase = _FakeLinear(4640, 512, input_size=2048)
assert not check_marlin_supports_layer(layer, 128)
assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True)
# A group straddling the TP shard cannot be fixed by padding
layer = _FakeLinear(4608, 4672, input_size=18688)
assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
@pytest.mark.skipif(
_gpu_marlin_unsupported(),
reason="Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("group_size", [-1, 32])
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
"""Pad a tile-misaligned MoE intermediate the way the WNA16 Marlin MoE prep
does, run the real repack + fused_marlin_moe, and check against the
dequantized reference. Symmetric int4's quantized zero decodes to -8, so the
padded region only stays out of the output via the zero-padded scales.
"""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
_pad_rows,
_pad_w13_shard_cols,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_padded_intermediate,
marlin_moe_permute_scales,
)
n, k, e = shape
topk, m = 2, 33
padded_n = marlin_moe_padded_intermediate(n, group_size)
assert padded_n != n, "test should exercise padding"
dtype = torch.float16
device = torch.device("cuda")
quant_type = scalar_types.uint4b8
bits = quant_type.size_bits
pack = 32 // bits
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
def quant(w, size_k, size_n):
# w is (size_n, size_k); gptq expects (size_k, size_n).
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
return ref, gptq_pack(q_w, bits, size_k, size_n), s
w13_qw, w13_s, w13_ref = [], [], []
w2_qw, w2_s, w2_ref = [], [], []
for i in range(e):
ref, qw, s = quant(w1[i], k, 2 * n)
w13_ref.append(ref.T) # (2n, k)
w13_qw.append(qw)
w13_s.append(s)
ref, qw, s = quant(w2[i], n, k)
w2_ref.append(ref.T) # (k, n)
w2_qw.append(qw)
w2_s.append(s)
w13_qweight = torch.stack(w13_qw)
w2_qweight = torch.stack(w2_qw)
w13_scales = torch.stack(w13_s)
w2_scales = torch.stack(w2_s)
w1_ref = torch.stack(w13_ref) # (e, 2n, k)
w2_ref = torch.stack(w2_ref) # (e, k, n)
# Pad the intermediate via the production helpers.
w13_qweight = _pad_w13_shard_cols(w13_qweight, n, padded_n)
w2_qweight = _pad_rows(w2_qweight, padded_n // pack)
w13_scales = _pad_w13_shard_cols(w13_scales, n, padded_n)
if group_size > 0:
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
marlin_w13 = ops.gptq_marlin_moe_repack(
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
)
marlin_w2 = ops.gptq_marlin_moe_repack(
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
)
group_or_pack = group_size if group_size != -1 else pack
marlin_w13_s = marlin_moe_permute_scales(
s=w13_scales, size_k=n, size_n=w13_scales.shape[2], group_size=group_size
)
marlin_w2_s = marlin_moe_permute_scales(
s=w2_scales,
size_k=w2_scales.shape[1] * group_or_pack,
size_n=w2_scales.shape[2],
group_size=group_size,
)
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
marlin_out = fused_marlin_moe(
a,
marlin_w13,
marlin_w2,
None,
None,
marlin_w13_s,
marlin_w2_s,
topk_weights,
topk_ids,
quant_type_id=quant_type.id,
global_num_experts=e,
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
w1_ref,
w2_ref,
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(marlin_out, ref, atol=5e-2, rtol=0)
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="MoE Marlin is not selected on ROCm.",
)
def test_check_moe_marlin_supports_layer_padding():
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
check_moe_marlin_supports_layer,
)
def make_layer(hidden, intermediate):
layer = SimpleNamespace()
layer.hidden_size = hidden
layer.apply_router_weight_on_input = False
layer.moe_config = SimpleNamespace(
hidden_dim=hidden, intermediate_size_per_partition_unpadded=intermediate
)
return layer
# group=32 with intermediate % 64 != 0: rejected strictly, accepted w/ padding
layer = make_layer(4096, 96)
assert not check_moe_marlin_supports_layer(layer, 32)
assert check_moe_marlin_supports_layer(layer, 32, allow_tile_padding=True)
# channelwise misaligned intermediate is paddable
assert check_moe_marlin_supports_layer(layer, -1, allow_tile_padding=True)
# A group straddling the boundary cannot be fixed by padding
layer = make_layer(4096, 176)
assert not check_moe_marlin_supports_layer(layer, 128, allow_tile_padding=True)
# hidden_size is the MoE I/O extent and is never padded
layer = make_layer(4090, 128)
assert not check_moe_marlin_supports_layer(layer, 64, allow_tile_padding=True)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("quant", ["channel", "tensor"])
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
real prepare + fused_marlin_moe against the dequantized reference."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_intermediate_size,
marlin_moe_padded_intermediate,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_fp8_moe_layer_for_marlin,
)
n, k, e = shape
topk, m = 2, 33
fp8 = torch.float8_e4m3fn
dtype = torch.bfloat16
device = torch.device("cuda")
padded_n = marlin_moe_padded_intermediate(n, -1)
assert padded_n != n
def q(w): # (out, in) -> fp8 weight, scale, dequant reference
dim = None if quant == "tensor" else 1
s = (w.abs().amax(dim, keepdim=dim is not None) / 448.0).clamp(min=1e-8)
wq = (w / s).clamp(-448, 448).to(fp8)
ref = wq.to(dtype) * s.to(dtype)
s = s.reshape(1) if quant == "tensor" else s.squeeze(1)
return wq, s, ref
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
w13_q, w13_s, w1_ref = zip(*(q(w1[i]) for i in range(e)))
w2_q, w2_s, w2_ref = zip(*(q(w2[i]) for i in range(e)))
w13_weight, w2_weight = torch.stack(w13_q), torch.stack(w2_q)
layer = SimpleNamespace(
num_experts=e,
hidden_size=k,
intermediate_size_per_partition=n,
orig_dtype=dtype,
w13_weight=w13_weight,
)
pw13, pw2, ps13, ps2 = prepare_fp8_moe_layer_for_marlin(
layer, w13_weight, w2_weight, torch.stack(w13_s), torch.stack(w2_s)
)
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
out = fused_marlin_moe(
a,
pw13,
pw2,
None,
None,
ps13,
ps2,
topk_weights,
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
workspace=get_marlin_workspace(device),
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
torch.stack(w1_ref),
torch.stack(w2_ref),
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
@pytest.mark.skipif(
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
reason="FP8 Marlin is not supported on this GPU type.",
)
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_mxfp8_marlin_moe_padded_round_trip(shape):
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
unit e8m0 scales so the reference is the exact fp8 dequant."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
fused_marlin_moe,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_moe_intermediate_size,
marlin_moe_padded_intermediate,
)
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
prepare_mxfp8_moe_layer_for_marlin,
)
n, k, e = shape
topk, m, gs, e8m0_one = 2, 33, 32, 127
fp8 = torch.float8_e4m3fn
dtype = torch.bfloat16
device = torch.device("cuda")
padded_n = marlin_moe_padded_intermediate(n, gs)
assert padded_n != n
a = torch.randn((m, k), device=device, dtype=dtype) / 10
w13_weight = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
w2_weight = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
w13_weight = w13_weight.clamp(-448, 448).to(fp8)
w2_weight = w2_weight.clamp(-448, 448).to(fp8)
w13_scale = torch.full(
(e, 2 * n, k // gs), e8m0_one, dtype=torch.uint8, device=device
)
w2_scale = torch.full((e, k, n // gs), e8m0_one, dtype=torch.uint8, device=device)
layer = SimpleNamespace(
num_experts=e, hidden_size=k, intermediate_size_per_partition=n
)
with set_current_vllm_config(VllmConfig()):
pw13, pw2, ps13, ps2 = prepare_mxfp8_moe_layer_for_marlin(
layer, w13_weight, w2_weight, w13_scale, w2_scale
)
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
score = torch.randn((m, e), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
out = fused_marlin_moe(
a,
pw13,
pw2,
None,
None,
ps13,
ps2,
topk_weights,
topk_ids,
quant_type_id=scalar_types.float8_e4m3fn.id,
global_num_experts=e,
workspace=get_marlin_workspace(device),
)
with set_current_vllm_config(VllmConfig()):
ref = torch_experts(
a,
w13_weight.to(dtype),
w2_weight.to(dtype),
topk_weight=topk_weights,
topk_ids=topk_ids,
global_num_experts=e,
)
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
@@ -1,322 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MXFP4 linear kernel selection logic (CPU-only).
Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`.
"""
from unittest.mock import MagicMock, patch
import pytest
import torch
import vllm.envs as envs
from vllm.model_executor.kernels.linear import (
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
FlashInferMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
MxFp4LinearKernel,
MxFp4LinearLayerConfig,
XPUMxFp4LinearKernel,
init_mxfp4_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
kMxfp6E2M3Dynamic,
kMxfp6E3M2Dynamic,
)
from vllm.platforms import PlatformEnum
pytestmark = pytest.mark.cpu_test
# Kernels that quantize activations themselves (true W4A4): they require an
# explicit MXFP4-dynamic activation key.
_TRUE_W4A4_KERNELS = [
FlashInferMxFp4LinearKernel,
XPUMxFp4LinearKernel,
AiterMxfp4LinearKernel,
]
# Weight-only (A16) kernels: they never quantize activations. They still accept
# MXFP4 activation keys as an intentional compatibility fallback.
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel]
def _make_emulation_layer():
layer = torch.nn.Module()
layer.register_parameter(
"weight",
torch.nn.Parameter(torch.ones((2, 2), dtype=torch.uint8), False),
)
layer.register_parameter(
"weight_scale",
torch.nn.Parameter(torch.ones((2, 1), dtype=torch.uint8), False),
)
return layer
def test_can_implement_is_abstract():
"""Test that can_implement()/is_supported() are properly defined."""
assert hasattr(MxFp4LinearKernel, "can_implement")
assert hasattr(MxFp4LinearKernel, "is_supported")
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
def test_true_w4a4_kernels_accept_dynamic_mxfp4_activation(kernel_cls):
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
can_implement, reason = kernel_cls.can_implement(config)
assert can_implement, reason
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
def test_true_w4a4_kernels_reject_unset_activation(kernel_cls):
"""None means weight-only/unquantized activations, not dynamic MXFP4."""
config = MxFp4LinearLayerConfig()
can_implement, reason = kernel_cls.can_implement(config)
assert not can_implement
assert reason
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
def test_true_w4a4_kernels_reject_explicit_non_mxfp4_activation(kernel_cls):
"""FlashInfer/XPU/Aiter quantize activations to MXFP4 internally, so an
explicit request for a different activation format must be rejected."""
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
can_implement, reason = kernel_cls.can_implement(config)
assert not can_implement
assert reason
@pytest.mark.parametrize("kernel_cls", _WEIGHT_ONLY_KERNELS)
@pytest.mark.parametrize("activation_quant_key", [None, kMxfp4Dynamic])
def test_weight_only_kernels_accept_unquantized_or_mxfp4_activation(
kernel_cls, activation_quant_key
):
"""Marlin/Humming never quantize activations, so an unset activation key,
or one that already describes MXFP4-shaped data, is tolerated. When an
activation key is explicitly set, a warning must be logged noting that it
is ignored, since these kernels are weight-only (A16)."""
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
with patch(f"{kernel_cls.__module__}.logger.warning_once") as warning_once:
can_implement, reason = kernel_cls.can_implement(config)
assert can_implement, reason
if activation_quant_key is None:
warning_once.assert_not_called()
else:
warning_once.assert_called_once()
message = warning_once.call_args.args[0]
assert "the requested activation quantization" in message
assert "is ignored" in message
@pytest.mark.parametrize("kernel_cls", _WEIGHT_ONLY_KERNELS)
def test_weight_only_kernels_reject_non_mxfp4_activation(kernel_cls):
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
can_implement, reason = kernel_cls.can_implement(config)
assert not can_implement
assert reason
@pytest.mark.parametrize(
"activation_quant_key",
[None, kMxfp4Dynamic, kMxfp6E3M2Dynamic, kMxfp6E2M3Dynamic],
)
def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
"""EmulationMxfp4LinearKernel is the universal fallback: it must accept
every supported activation format."""
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
# which we are not testing here.
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
with patch(
"vllm.model_executor.kernels.linear._get_linear_backend",
return_value="emulation",
):
can_implement, reason = EmulationMxfp4LinearKernel.can_implement(config)
assert can_implement, reason
def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
"""quant_dequant_func must be derived purely from the config's activation
QuantKey, not set externally."""
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
# which we are not testing here.
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
with patch(
"vllm.model_executor.kernels.linear._get_linear_backend",
return_value="emulation",
):
weight_only_config = MxFp4LinearLayerConfig()
kernel = EmulationMxfp4LinearKernel(weight_only_config)
x = torch.randn(4)
# identity for weight-only
assert torch.equal(kernel.quant_dequant_func(x), x)
w4a4_config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
kernel = EmulationMxfp4LinearKernel(w4a4_config)
assert kernel.quant_dequant_func is quant_dequant_mxfp4
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_emulation_kernel_dequantizes_at_load_and_keeps_activation_qdq(
dtype, monkeypatch
):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
kernel = EmulationMxfp4LinearKernel(
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
)
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
layer = _make_emulation_layer()
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
x = torch.tensor([[2.0, 4.0]], dtype=dtype)
with (
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", True),
patch(
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
return_value=dequantized,
) as dequant,
):
kernel.process_weights_after_loading(layer)
actual = kernel.apply_weights(layer, x)
expected = torch.nn.functional.linear(x * 0.5, dequantized.to(dtype))
torch.testing.assert_close(actual, expected)
dequant.assert_called_once()
kernel.quant_dequant_func.assert_called_once_with(x)
assert layer.weight.dtype == torch.bfloat16
assert layer.weight.device == dequantized.device
assert not layer.weight.requires_grad
assert layer.weight_scale is not None
assert not layer.weight_scale.requires_grad
def test_emulation_kernel_opt_out_dequantizes_per_invocation(monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
kernel = EmulationMxfp4LinearKernel(
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
)
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
layer = _make_emulation_layer()
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16)
with (
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", False),
patch(
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
return_value=dequantized,
) as dequant,
):
kernel.process_weights_after_loading(layer)
dequant.assert_not_called()
first = kernel.apply_weights(layer, x)
second = kernel.apply_weights(layer, x)
expected = torch.nn.functional.linear(x * 0.5, dequantized)
torch.testing.assert_close(first, expected)
torch.testing.assert_close(second, expected)
assert dequant.call_count == 2
assert kernel.quant_dequant_func.call_count == 2
assert layer.weight.dtype == torch.uint8
assert layer.weight_scale is not None
def test_aiter_kernel_is_supported_requires_native_mx_support():
"""AiterMxfp4LinearKernel must not be selected on platforms without
native MX compute, even if AITER itself is importable."""
with patch(
"vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform.supports_mx",
return_value=False,
):
is_supported, reason = AiterMxfp4LinearKernel.is_supported()
assert not is_supported
assert reason
class OOTMxFp4LinearKernel(MxFp4LinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
pass
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_init_mxfp4_linear_kernel_dispatches_to_registered_kernel(platform_mock):
"""init_mxfp4_linear_kernel should select a registered kernel that
reports itself as supported, and construct it with a fresh config."""
platform_mock._enum = PlatformEnum.OOT
register_linear_kernel(OOTMxFp4LinearKernel, PlatformEnum.OOT, "mxfp4")
kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
assert isinstance(kernel, OOTMxFp4LinearKernel)
assert kernel.config == MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
class UnsupportedMxFp4LinearKernel(MxFp4LinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return False, "never supported"
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
pass
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_init_mxfp4_linear_kernel_raises_when_no_kernel_matches(platform_mock):
platform_mock._enum = PlatformEnum.UNSPECIFIED
register_linear_kernel(
UnsupportedMxFp4LinearKernel, PlatformEnum.UNSPECIFIED, "mxfp4"
)
with pytest.raises(ValueError, match="Failed to find a kernel"):
init_mxfp4_linear_kernel()
@@ -1,304 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Copyright (C) 2025 Roberto L. Castro (Roberto.LopezCastro@ist.ac.at).
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import numpy as np
import pytest
import torch
from compressed_tensors.transform.utils.hadamard import deterministic_hadamard_matrix
from vllm._custom_ops import fusedQuantizeMx, matmul_mxf4_bf16_tn
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not torch.cuda.is_available():
pytest.skip("CUDA required for these tests.", allow_module_level=True)
if not (
current_platform.has_device_capability(100)
or current_platform.has_device_capability(120)
):
pytest.skip(
reason="Tests require compute capability 10.0 (100) or 12.0 (120).",
allow_module_level=True,
)
# ----- Helpers -----
def get_hadamard_matrix(group_size: int, dtype: torch.dtype, device: torch.device):
return (
deterministic_hadamard_matrix(group_size, dtype=dtype, device=device)
* group_size**-0.5
)
def _rtne_fp4(x: torch.Tensor):
device = x.device
grid = torch.tensor(
[
-6.0,
-4.0,
-3.0,
-2.0,
-1.5,
-1.0,
-0.5,
-0.0,
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
],
dtype=x.dtype,
device=x.device,
)
grid_int = torch.tensor(
[-1, -2, -3, -4, -5, -6, -7, -8, 0, 1, 2, 3, 4, 5, 6, 7],
dtype=torch.uint8,
device=device,
)
inds = torch.bucketize(x, grid)
lo, hi = (inds - 1).clamp(min=0, max=15), inds.clamp(min=0, max=15)
g_lo, g_hi = grid[lo], grid[hi]
pick_hi = (g_hi - x < x - g_lo) | (g_hi - x == x - g_lo) & (grid_int[hi] % 2 == 0)
y = torch.where(pick_hi, g_hi, g_lo)
y_int = torch.where(pick_hi, grid_int[hi], grid_int[lo])
y_int_packed = (y_int[..., 1::2] & 0xF) << 4 | y_int[..., ::2] & 0xF
return y, y_int_packed
def _dq_fp4(x_e2m1: torch.Tensor, x_e8m0: torch.Tensor, alpha: float):
device = x_e2m1.device
x_e2m1_i32 = x_e2m1.view(dtype=torch.uint8).to(dtype=torch.int32)
x_e2m1_unpacked = torch.stack(
[x_e2m1_i32 & 0xF, (x_e2m1_i32 >> 4) & 0xF], dim=-1
).flatten(start_dim=-2)
grid_dq = torch.tensor(
[
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
],
dtype=torch.float64,
device=device,
)
x_fp4_dq = grid_dq[x_e2m1_unpacked]
scales_dq = x_e8m0.to(torch.float64)
x_dq = (x_fp4_dq.unflatten(dim=-1, sizes=(-1, 32)) * scales_dq[..., None]).flatten(
start_dim=-2
) / alpha
return x_dq, x_fp4_dq, scales_dq
def _unpack_mask(clip_mask: torch.Tensor) -> torch.Tensor:
clip_mask_unpacked_dq = torch.zeros(
*clip_mask.shape[:-1],
clip_mask.size(-1) * 8,
dtype=torch.bool,
device=clip_mask.device,
)
for i in range(8):
clip_mask_unpacked_dq[..., i::8] = (clip_mask >> i) & 1
return clip_mask_unpacked_dq
def _forward_quantize_ref(
x: torch.Tensor, h: torch.Tensor, rot_size: int, quest: bool = True
):
device = x.device
xh_ref64 = (
x.unflatten(dim=-1, sizes=(-1, rot_size)).to(dtype=torch.float64)
@ h.reshape(rot_size, rot_size).to(dtype=torch.float64)
).flatten(start_dim=-2)
if quest:
scales_ref64_ = (
xh_ref64.unflatten(dim=-1, sizes=(-1, 32)).std(dim=-1, correction=0)
* (2.92247856 / 6.0)
+ 1e-8
)
else:
abs_max = xh_ref64.unflatten(dim=-1, sizes=(-1, 32)).abs().amax(dim=-1)
scales_ref64_ = abs_max + 1e-8
xh_e8m0_ref = scales_ref64_.log2().floor().exp2().to(dtype=torch.float8_e8m0fnu)
scales_ref64 = xh_e8m0_ref.to(dtype=torch.float64)
xh_scaled_ref64 = (
xh_ref64.unflatten(dim=-1, sizes=(-1, 32)) / scales_ref64[..., None]
).flatten(start_dim=-2)
if not quest:
xh_scaled_ref64 *= 3
clip_mask_unpacked_ref = xh_scaled_ref64.abs() < 6.0
clip_mask_ref = torch.zeros(
*x.shape[:-1], x.size(-1) // 8, dtype=torch.uint8, device=device
)
for i in range(8):
clip_mask_ref |= clip_mask_unpacked_ref[..., i::8].to(dtype=torch.uint8) << i
xh_fp4_ref, xh_e2m1_ref = _rtne_fp4(xh_scaled_ref64)
xh_dq, xh_fp4_dq, scales_dq = _dq_fp4(
xh_e2m1_ref, xh_e8m0_ref, alpha=1.0 if quest else 3.0
)
clip_mask_unpacked_dq = _unpack_mask(clip_mask_ref)
assert xh_fp4_dq.equal(xh_fp4_ref)
assert scales_dq.equal(scales_ref64)
assert clip_mask_unpacked_dq.equal(clip_mask_unpacked_ref)
return (
xh_dq,
clip_mask_unpacked_ref,
(xh_e2m1_ref, xh_e8m0_ref, clip_mask_ref),
)
DTYPE = torch.bfloat16
DEVICE = torch.device("cuda:0")
ROT_SIZES = [32, 64, 128]
SEEDS = [0]
BATCHES = [1, 16]
LLAMA_MODELS = {
"7B": [(4096, 3 * 4096), (4096, 4096), (4096, 2 * 10752), (10752, 4096)],
"13B": [(5120, 3 * 5120), (5120, 5120), (5120, 2 * 13568), (13568, 5120)],
"33B": [(6656, 3 * 6656), (6656, 6656), (6656, 2 * 17664), (17664, 6656)],
"70B": [(8192, 3 * 8192), (8192, 8192), (8192, 2 * 21760), (21760, 8192)],
}
@pytest.fixture(autouse=True)
def _seed_each_test():
set_random_seed(0)
np.random.seed(0)
torch.random.manual_seed(0)
@pytest.mark.parametrize("rot_size", ROT_SIZES)
@torch.inference_mode()
def test_fused_quantization_absmax(rot_size: int):
dtype, device = DTYPE, DEVICE
h = get_hadamard_matrix(rot_size, dtype, device)
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size, quest=False)
xh_e2m1, xh_e8m0 = fusedQuantizeMx(x, h, method="abs_max")
xh_e8m0 = xh_e8m0.reshape(2, 4096, 4096 // 32)
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e8m0, alpha=3.0)
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-4
m, n, k = 1, 504, 4096
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="abs_max")
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="abs_max")
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
out_ref = a_dq @ b_dq.transpose(-2, -1)
a_scale_block = to_blocked(a_e8m0, backend="triton")
b_scale_block = to_blocked(b_e8m0, backend="triton")
alpha = torch.tensor([1.0], device=device)
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
assert out.equal(out_ref.to(dtype=out.dtype))
@pytest.mark.parametrize("rot_size", ROT_SIZES)
@torch.inference_mode()
def test_fused_quantization_quest(rot_size: int):
dtype, device = DTYPE, DEVICE
h = get_hadamard_matrix(rot_size, dtype, device)
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size, quest=True)
xh_e2m1, xh_e8m0 = fusedQuantizeMx(x, h, method="quest")
xh_e8m0 = xh_e8m0.reshape(2, 4096, 4096 // 32)
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e8m0, alpha=1.0)
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-4
m, n, k = 504, 504, 2048
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="quest")
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="quest")
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
out_ref = a_dq @ b_dq.transpose(-2, -1)
a_scale_block = to_blocked(a_e8m0, backend="triton")
b_scale_block = to_blocked(b_e8m0, backend="triton")
alpha = torch.tensor([1.0], device=device)
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
assert out.equal(out_ref.to(dtype=out.dtype))
@pytest.mark.parametrize("model", list(LLAMA_MODELS.keys()))
@pytest.mark.parametrize("layer_idx", [0, 1, 2, 3])
@pytest.mark.parametrize("batch", [1, 16])
@pytest.mark.parametrize("had_size", ROT_SIZES)
@torch.inference_mode()
def test_llama_shapes(model: str, layer_idx: int, batch: int, had_size: int):
dtype, device = DTYPE, DEVICE
m = batch
k, n = LLAMA_MODELS[model][layer_idx]
h = get_hadamard_matrix(had_size, dtype, device)
a = torch.rand(m, k, dtype=dtype, device=device) * 25.0
b = torch.rand(n, k, dtype=dtype, device=device) * 25.0
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="quest")
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="quest")
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
out_ref = a_dq @ b_dq.transpose(-2, -1)
a_scale_block = to_blocked(a_e8m0, backend="triton")
b_scale_block = to_blocked(b_e8m0, backend="triton")
alpha = torch.tensor([1.0], device=device)
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
assert out.equal(out_ref.to(dtype=out.dtype))
@@ -1,82 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests that triton_kernel_moe_forward correctly applies expert_map
remapping when expert parallelism (EP) is enabled.
When expert_map is provided, global expert IDs are remapped to local IDs
via topk + expert_map remap + make_routing_data before building routing
structures, and the expert_map passed downstream to triton_kernel_fused_experts
is None (already applied).
"""
from unittest.mock import MagicMock, patch
import torch
class TestTritonMoeForwardExpertMap:
"""Test that triton_kernel_moe_forward applies expert_map remapping
when expert_map is provided (EP active)."""
def test_expert_map_remap(self):
device = "cuda" if torch.cuda.is_available() else "cpu"
mock_expert_map = torch.tensor([0, -1, 1, -1], device=device)
from vllm.utils.import_utils import import_triton_kernels
import_triton_kernels()
mock_routing_data = MagicMock()
mock_gather = MagicMock()
mock_scatter = MagicMock()
with (
patch("triton_kernels.topk.topk") as mock_topk,
patch(
"vllm.model_executor.layers.fused_moe.experts."
"gpt_oss_triton_kernels_moe.make_routing_data"
) as mock_make_routing,
patch(
"vllm.model_executor.layers.fused_moe.experts."
"gpt_oss_triton_kernels_moe.triton_kernel_fused_experts"
) as mock_fused_experts,
):
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
triton_kernel_moe_forward,
)
sparse_result = MagicMock()
sparse_result.indx = torch.tensor([[0, 2]], dtype=torch.int32)
sparse_result.vals = torch.tensor([[0.6, 0.4]])
mock_topk.return_value = sparse_result
mock_make_routing.return_value = (
mock_routing_data,
mock_gather,
mock_scatter,
)
mock_fused_experts.return_value = torch.zeros((1, 8), device=device)
hidden = torch.randn((1, 8), device=device)
w1 = torch.randn((2, 8, 16), device=device)
w2 = torch.randn((2, 8, 8), device=device)
logits = torch.randn((1, 4), device=device)
triton_kernel_moe_forward(
hidden_states=hidden,
w1=w1,
w2=w2,
gating_output=logits,
topk=2,
renormalize=True,
expert_map=mock_expert_map,
)
mock_topk.assert_called_once()
mock_make_routing.assert_called_once()
# expert_map should be None in the fused_experts call
# (already applied).
call_kwargs = mock_fused_experts.call_args
assert call_kwargs[1].get("expert_map") is None or (len(call_kwargs[0]) > 0)
@@ -1,150 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MXFP6 linear kernel selection logic (CPU-only).
Run `pytest tests/kernels/quantization/test_mxfp6_kernel_selection.py`.
"""
from unittest.mock import patch
import pytest
import torch
from vllm.model_executor.kernels.linear import (
EmulationMxfp6LinearKernel,
MxFp6LinearKernel,
MxFp6LinearLayerConfig,
init_mxfp6_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from vllm.platforms import PlatformEnum
pytestmark = pytest.mark.cpu_test
# The only implementation available at the moment is software emulation.
_WEIGHT_QUANT_KEYS = [kMxfp6E3M2Static, kMxfp6E2M3Static]
def test_can_implement_is_abstract():
"""Test that can_implement()/is_supported() are properly defined."""
assert hasattr(MxFp6LinearKernel, "can_implement")
assert hasattr(MxFp6LinearKernel, "is_supported")
def test_emulation_kernel_rejects_non_mxfp6_weights():
"""EmulationMxfp6LinearKernel must not implement a non-MXFP6 weight
format."""
config = MxFp6LinearLayerConfig(weight_quant_key=kMxfp4Static)
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
assert not can_implement
assert reason
@pytest.mark.parametrize("weight_quant_key", _WEIGHT_QUANT_KEYS)
@pytest.mark.parametrize(
"activation_quant_key",
[None, kMxfp4Dynamic, kMxfp6E3M2Dynamic, kMxfp6E2M3Dynamic],
)
def test_emulation_kernel_accepts_any_supported_config(
weight_quant_key, activation_quant_key
):
"""EmulationMxfp6LinearKernel is the only backend today: it must accept
every supported weight/activation format combination."""
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key
)
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
assert can_implement, reason
@pytest.mark.parametrize("weight_quant_key", _WEIGHT_QUANT_KEYS)
def test_emulation_kernel_rejects_non_mxfp4_or_mxfp6_activation(weight_quant_key):
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key, activation_quant_key=kMxfp4Static
)
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
assert not can_implement
assert reason
class OOTMxFp6LinearKernel(MxFp6LinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
pass
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_init_mxfp6_linear_kernel_dispatches_to_registered_kernel(platform_mock):
"""init_mxfp6_linear_kernel should select a registered kernel that
reports itself as supported/able to implement the given config, and
construct it with that exact config."""
platform_mock._enum = PlatformEnum.OOT
register_linear_kernel(OOTMxFp6LinearKernel, PlatformEnum.OOT, "mxfp6")
kernel = init_mxfp6_linear_kernel(
weight_quant_key=kMxfp6E3M2Static, activation_quant_key=kMxfp6E3M2Dynamic
)
assert isinstance(kernel, OOTMxFp6LinearKernel)
assert kernel.config == MxFp6LinearLayerConfig(
weight_quant_key=kMxfp6E3M2Static, activation_quant_key=kMxfp6E3M2Dynamic
)
class UnsupportedMxFp6LinearKernel(MxFp6LinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return False, "never supported"
@classmethod
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
pass
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_init_mxfp6_linear_kernel_raises_when_no_kernel_matches(platform_mock):
platform_mock._enum = PlatformEnum.UNSPECIFIED
register_linear_kernel(
UnsupportedMxFp6LinearKernel, PlatformEnum.UNSPECIFIED, "mxfp6"
)
with pytest.raises(ValueError, match="Failed to find a kernel"):
init_mxfp6_linear_kernel(weight_quant_key=kMxfp6E3M2Static)
@@ -1,135 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MXFP8 linear kernel selection logic (CPU-only)
Run `pytest tests/kernels/quantization/test_mxfp8_kernel_selection.py`.
"""
from contextlib import contextmanager
from unittest.mock import patch
import pytest
from vllm.model_executor.kernels.linear import (
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
MarlinMxfp8LinearKernel,
Mxfp8LinearLayerConfig,
init_mxfp8_linear_kernel,
)
from vllm.platforms import PlatformEnum
pytestmark = pytest.mark.cpu_test
# Kernels backed by FlashInfer mm_mxfp8, which requires N, K >= 128 and
# K % 32 == 0.
_MM_MXFP8_KERNELS = [
FlashInferCutedslMxfp8LinearKernel,
FlashInferCutlassMxfp8LinearKernel,
]
_SUPPORTED_SHAPE = (4096, 4096)
# (N, K): N < 128, K < 128, K % 32 != 0.
_UNSUPPORTED_SHAPES = [(64, 4096), (4096, 64), (4096, 4112)]
# (N, K) with N < 128 or K < 128, which only Marlin runs.
_MARLIN_ONLY_SHAPES = [(32, 256), (64, 256), (96, 256), (256, 64)]
# (N, K): N <= 0, K <= 0, K % 32 != 0.
_MARLIN_UNSUPPORTED_SHAPES = [(0, 256), (256, 0), (256, 100)]
_SELECTION_CASES = [
(100, (128, 128), FlashInferCutedslMxfp8LinearKernel),
(100, (130, 256), FlashInferCutedslMxfp8LinearKernel),
(103, (128, 128), FlashInferCutedslMxfp8LinearKernel),
(103, (130, 256), FlashInferCutedslMxfp8LinearKernel),
(120, (128, 128), FlashInferCutlassMxfp8LinearKernel),
(120, (130, 256), MarlinMxfp8LinearKernel),
(121, (128, 128), FlashInferCutlassMxfp8LinearKernel),
(121, (130, 256), MarlinMxfp8LinearKernel),
*[
(capability, shape, MarlinMxfp8LinearKernel)
for capability in (100, 103, 120, 121)
for shape in _MARLIN_ONLY_SHAPES
],
]
@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS)
def test_mm_mxfp8_kernels_accept_supported_shape(kernel_cls):
config = Mxfp8LinearLayerConfig(weight_shape=_SUPPORTED_SHAPE)
can_implement, reason = kernel_cls.can_implement(config)
assert can_implement, reason
@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS)
@pytest.mark.parametrize("weight_shape", _UNSUPPORTED_SHAPES)
def test_mm_mxfp8_kernels_reject_unsupported_shape(kernel_cls, weight_shape):
config = Mxfp8LinearLayerConfig(weight_shape=weight_shape)
can_implement, reason = kernel_cls.can_implement(config)
assert not can_implement
assert reason
@pytest.mark.parametrize("weight_shape", _MARLIN_UNSUPPORTED_SHAPES)
def test_marlin_mxfp8_rejects_unsupported_shape(weight_shape):
config = Mxfp8LinearLayerConfig(weight_shape=weight_shape)
can_implement, reason = MarlinMxfp8LinearKernel.can_implement(config)
assert not can_implement
assert reason
@contextmanager
def _patch_cuda_platform(capability: int):
"""Patch the platform so kernel selection runs as on a CUDA GPU with the
given compute capability. The FlashInfer and Marlin kernels report as
supported, CuTe-DSL only on SM100/103."""
with (
patch("vllm.model_executor.kernels.linear.current_platform") as platform,
patch(
"vllm.model_executor.kernels.linear.mxfp8.flashinfer.current_platform."
"is_device_capability_family",
side_effect=lambda cap, device_id=0: cap // 10 == capability // 10,
),
patch.object(
FlashInferCutedslMxfp8LinearKernel,
"is_supported",
return_value=(capability in (100, 103), None),
),
patch.object(
FlashInferCutlassMxfp8LinearKernel,
"is_supported",
return_value=(True, None),
),
patch.object(
MarlinMxfp8LinearKernel, "is_supported", return_value=(True, None)
),
):
platform._enum = PlatformEnum.CUDA
yield
@pytest.mark.parametrize(
("capability", "expected"),
[(100, True), (103, True), (120, False), (121, False)],
)
def test_cutlass_mxfp8_requires_aligned_n_only_on_sm12x(capability, expected):
"""(130, 256) passes the generic mm_mxfp8 limits. Only the SM12x CUTLASS
backend needs N % 32 == 0, so SM100/103 must keep accepting it."""
config = Mxfp8LinearLayerConfig(weight_shape=(130, 256))
with _patch_cuda_platform(capability):
can_implement, _ = FlashInferCutlassMxfp8LinearKernel.can_implement(config)
assert can_implement == expected
@pytest.mark.parametrize(
("capability", "weight_shape", "expected_kernel_cls"), _SELECTION_CASES
)
def test_init_mxfp8_linear_kernel(capability, weight_shape, expected_kernel_cls):
"""A layer that mm_mxfp8 cannot handle must fall through to the next
kernel in the CUDA priority list instead of being selected and failing
in apply_weights."""
with _patch_cuda_platform(capability):
kernel = init_mxfp8_linear_kernel(weight_shape=weight_shape)
assert isinstance(kernel, expected_kernel_cls)
assert kernel.config == Mxfp8LinearLayerConfig(weight_shape=weight_shape)
@@ -1,866 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import cast
import pytest
import torch
from nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
break_fp4_bytes,
kE2M1ToFloat,
)
from safetensors import safe_open
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import KernelConfig
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
EmulationNvFp4LinearKernel,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
Nvfp4QuantizationEmulationTritonExperts,
)
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
from vllm.model_executor.layers.quantization.utils import (
nvfp4_emulation_utils,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_to_dtype,
ref_nvfp4_quant_dequant,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.triton_utils import triton
from vllm.utils.torch_utils import set_random_seed
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950
else:
def on_gfx950() -> bool:
return False
MOE_MODEL_CONFIGS = {
"nvidia/Qwen3-30B-A3B-NVFP4": {
"shards": ["model-00001-of-00004.safetensors"],
"expert_prefix": "model.layers.9.mlp.experts.",
# Position of the expert index in the dot-split key.
"expert_idx_pos": 5,
}
}
@pytest.fixture(scope="module")
def loaded_model_files():
return {
model_id: hf_api().snapshot_download(
repo_id=model_id, allow_patterns=config["shards"]
)
for model_id, config in MOE_MODEL_CONFIGS.items()
}
class Nvfp4QuantizationEmulationTritonExpertsReference(TritonExperts):
"""Extension of TritonExperts to support emulated NVFP4 MoE experts.
It may be used for NVFP4 models when the device does not have
native support for this dtype.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# `TritonExperts.apply` expects pre-dequantized weights,
# which we handle in `apply` below.
self.w1_scale_val = self.quant_config.w1_scale
self.w2_scale_val = self.quant_config.w2_scale
self.quant_config._w1.scale = None
self.quant_config._w2.scale = None
self.quantization_emulation = True
@property
def quant_dtype(self) -> torch.dtype | str | None:
return "nvfp4"
@property
def a1_scale(self) -> torch.Tensor | None:
return self.quant_config.a1_gscale
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kNvfp4Static, kNvfp4Dynamic)
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
assert w1.dtype == torch.uint8
assert w2.dtype == torch.uint8
# Dequantize w1 from packed NVFP4 to fp16/bf16
w13_global_scale = self.quant_config.g1_alphas
w1_dequant = dequantize_to_dtype(
tensor_fp4=w1,
tensor_sf=self.w1_scale_val,
global_scale=w13_global_scale,
dtype=hidden_states.dtype,
block_size=16,
swizzle=False,
)
# Dequantize w2 from packed NVFP4 to fp16/bf16
w2_global_scale = self.quant_config.g2_alphas
w2_dequant = dequantize_to_dtype(
tensor_fp4=w2,
tensor_sf=self.w2_scale_val,
global_scale=w2_global_scale,
dtype=hidden_states.dtype,
block_size=16,
swizzle=False,
)
super().apply(
output=output,
hidden_states=hidden_states,
w1=w1_dequant,
w2=w2_dequant,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
a1q_scale=None,
a2_scale=self.quant_config.a2_gscale,
workspace13=workspace13,
workspace2=workspace2,
expert_tokens_meta=expert_tokens_meta,
apply_router_weight_on_input=apply_router_weight_on_input,
)
@pytest.mark.parametrize(
("config_kwargs", "expected_reason"),
[
({"has_bias": True}, "kernel does not support bias"),
({"is_lora_enabled": True}, "kernel does not support LoRA"),
],
)
def test_nvfp4_emulation_support_check_rejects_bias_and_lora(
config_kwargs: dict[str, bool],
expected_reason: str,
) -> None:
moe_config = FusedMoEConfig(
num_experts=2,
experts_per_token=1,
hidden_dim=16,
intermediate_size=16,
num_local_experts=2,
num_logical_experts=2,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.TopK,
**config_kwargs,
)
supported, reason = Nvfp4QuantizationEmulationTritonExperts.is_supported_config(
Nvfp4QuantizationEmulationTritonExperts,
moe_config,
kNvfp4Static,
kNvfp4Dynamic,
mk.FusedMoEActivationFormat.Standard,
)
assert not supported
assert reason == expected_reason
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Triton NVFP4 kernel requires CUDA.",
)
def test_triton_dequantize_nvfp4(monkeypatch, loaded_model_files) -> None:
"""Test the Triton dequantization kernel against the CPU reference
using real NVFP4 weights from a checkpoint.
Tests both 2D (attention projection) and 3D (stacked MoE experts).
"""
checkpoint_path = loaded_model_files["nvidia/Qwen3-30B-A3B-NVFP4"]
shards = cast(list[str], MOE_MODEL_CONFIGS["nvidia/Qwen3-30B-A3B-NVFP4"]["shards"])
shard_path = f"{checkpoint_path}/{shards[0]}"
block_size = 16
with safe_open(shard_path, framework="pt", device="cpu") as f:
all_keys = list(f.keys())
# 2D case: attention projection
tensor_fp4_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight")
tensor_sf_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale")
global_scale_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale_2")
# 3D case: stack ALL experts for layer 9 up_proj
expert_prefix = "model.layers.9.mlp.experts."
expert_indices = sorted(
int(key.split(".")[5])
for key in all_keys
if key.startswith(expert_prefix) and key.endswith(".up_proj.weight")
)
assert len(expert_indices) > 0
all_fp4 = []
all_sf = []
all_global_scale = []
for index in expert_indices:
name = f"{expert_prefix}{index}.up_proj"
all_fp4.append(f.get_tensor(f"{name}.weight"))
all_sf.append(f.get_tensor(f"{name}.weight_scale"))
all_global_scale.append(f.get_tensor(f"{name}.weight_scale_2"))
tensor_fp4_3d = torch.stack(all_fp4)
tensor_sf_3d = torch.stack(all_sf)
global_scale_3d = torch.stack(all_global_scale)
test_cases = [
("2D base", tensor_fp4_2d, tensor_sf_2d, global_scale_2d),
(
"2D 2x rows",
tensor_fp4_2d.repeat(2, 1),
tensor_sf_2d.repeat(2, 1),
global_scale_2d,
),
(
"2D 4x rows",
tensor_fp4_2d.repeat(4, 1),
tensor_sf_2d.repeat(4, 1),
global_scale_2d,
),
(
"2D 2x cols",
tensor_fp4_2d.repeat(1, 2),
tensor_sf_2d.repeat(1, 2),
global_scale_2d,
),
("3D base", tensor_fp4_3d, tensor_sf_3d, global_scale_3d),
(
"3D 2x experts",
tensor_fp4_3d.repeat(2, 1, 1),
tensor_sf_3d.repeat(2, 1, 1),
global_scale_3d.repeat(2),
),
(
"3D 2x rows",
tensor_fp4_3d.repeat(1, 2, 1),
tensor_sf_3d.repeat(1, 2, 1),
global_scale_3d,
),
(
"3D 2x cols",
tensor_fp4_3d.repeat(1, 1, 2),
tensor_sf_3d.repeat(1, 1, 2),
global_scale_3d,
),
]
quantiles = [0.5, 0.001, 0.999]
# Move the E2M1 lookup table to CUDA ahead of time, as would normally
# happen during model loading (process_weights_after_loading). Both the
# Triton and PyTorch reference paths run on CUDA.
nvfp4_emulation_utils.kE2M1ToFloat_handle.val = (
nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda()
)
for label, tensor_fp4, tensor_sf, global_scale in test_cases:
fp4_cuda = tensor_fp4.cuda()
sf_cuda = tensor_sf.cuda()
gs_cuda = global_scale.cuda()
# Triton path
triton_result = dequantize_to_dtype(
fp4_cuda,
sf_cuda,
gs_cuda,
torch.bfloat16,
block_size,
swizzle=False,
)
# Reference path (PyTorch ops on CUDA, Triton dispatch disabled)
with monkeypatch.context() as m:
m.setattr(
nvfp4_emulation_utils.current_platform,
"is_cuda_alike",
lambda: False,
)
reference = dequantize_to_dtype(
fp4_cuda,
sf_cuda,
gs_cuda,
torch.bfloat16,
block_size,
swizzle=False,
)
torch.testing.assert_close(triton_result, reference, atol=0, rtol=0)
# Benchmark
shape = list(tensor_fp4.shape)
def _triton_bench(
fp4_cuda=fp4_cuda,
scale_cuda=sf_cuda,
global_scale_cuda=gs_cuda,
block_size=block_size,
):
return dequantize_to_dtype(
fp4_cuda,
scale_cuda,
global_scale_cuda,
torch.bfloat16,
block_size,
swizzle=False,
)
triton_ms, triton_min, triton_max = triton.testing.do_bench(
_triton_bench, quantiles=quantiles
)
def _reference_bench(
fp4_cuda=fp4_cuda,
scale_cuda=sf_cuda,
global_scale_cuda=gs_cuda,
block_size=block_size,
):
with monkeypatch.context() as m2:
m2.setattr(
nvfp4_emulation_utils.current_platform,
"is_cuda_alike",
lambda: False,
)
dequantize_to_dtype(
fp4_cuda,
scale_cuda,
global_scale_cuda,
torch.bfloat16,
block_size,
swizzle=False,
)
ref_ms, ref_min, ref_max = triton.testing.do_bench(
_reference_bench, quantiles=quantiles
)
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
print(f" dequantize {label} {shape}:")
print(
f" triton: median={triton_ms:.3f}ms, "
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
)
print(
f" reference: median={ref_ms:.3f}ms, "
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
)
print(f" speedup: {speedup:.2f}x")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Triton NVFP4 kernel requires CUDA.",
)
@pytest.mark.parametrize(
"m, k",
[
(1, 16),
(1, 4096),
(2, 4096),
(4, 4096),
(8, 4096),
(16, 4096),
(24, 4096),
(32, 4096),
(1, 8192),
(2, 8192),
(4, 8192),
(8, 8192),
(16, 8192),
(24, 8192),
(32, 8192),
(1, 32),
(2, 48),
(7, 64),
(16, 128),
(33, 160),
(128, 256),
(256, 512),
(1024, 1024),
(5120, 2048),
(2048, 4096),
(4096, 7168),
(8192, 8192),
(128, 16384),
],
)
@pytest.mark.parametrize("global_scale_value", [0.5, 1.0, 0.001])
def test_triton_nvfp4_quant_dequant(
monkeypatch, m: int, k: int, global_scale_value: float
) -> None:
"""Test the Triton quant-dequant kernel against the CPU reference."""
block_size = 16
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
global_scale = torch.tensor(global_scale_value, dtype=torch.float32, device="cuda")
# Triton path
triton_result = ref_nvfp4_quant_dequant(x, global_scale, block_size)
# CPU reference path
with monkeypatch.context() as mp:
mp.setattr(
nvfp4_emulation_utils.current_platform,
"is_cuda_alike",
lambda: False,
)
reference = ref_nvfp4_quant_dequant(x.cpu(), global_scale.cpu(), block_size)
torch.testing.assert_close(triton_result.cpu(), reference, atol=0, rtol=0)
# Benchmark (both paths on CUDA tensors for fair comparison)
quantiles = [0.5, 0.001, 0.999]
def _triton_bench(
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
):
return ref_nvfp4_quant_dequant(
input_tensor, input_global_scale, input_block_size
)
triton_ms, triton_min, triton_max = triton.testing.do_bench(
_triton_bench, quantiles=quantiles
)
def _reference_bench(
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
):
with monkeypatch.context() as mp2:
mp2.setattr(
nvfp4_emulation_utils.current_platform,
"is_cuda_alike",
lambda: False,
)
ref_nvfp4_quant_dequant(input_tensor, input_global_scale, input_block_size)
ref_ms, ref_min, ref_max = triton.testing.do_bench(
_reference_bench, quantiles=quantiles
)
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
print(f" quant_dequant [{m}x{k}] gs={global_scale_value}:")
print(
f" triton: median={triton_ms:.3f}ms, "
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
)
print(
f" reference: median={ref_ms:.3f}ms, "
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
)
print(f" speedup: {speedup:.2f}x")
def _load_nvfp4_moe_weights(
model_files: dict[str, str],
model_id: str,
tensor_parallel_size: int,
max_experts: int | None = None,
):
"""Load and stack NVFP4 MoE weights from checkpoint shards.
Returns (w1, w1_scale, w1_gscale, w2, w2_scale, w2_gscale,
a1_gscale, a2_gscale, num_experts, hidden_dim,
intermediate_size).
When max_experts is set, only the first max_experts experts are loaded.
When tensor_parallel_size > 1, the N dimension of w1 and the K
dimension of w2 are narrowed to the first TP shard (simulating
column-parallel on w1 / row-parallel on w2).
"""
cfg = MOE_MODEL_CONFIGS[model_id]
shards = cast(list[str], cfg["shards"])
checkpoint_path = model_files[model_id]
expert_prefix = cfg["expert_prefix"]
idx_pos = cast(int, cfg["expert_idx_pos"])
# Collect all tensors across shards into a flat dict — an expert's
# tensors may be split across multiple shard files.
all_tensors: dict[str, torch.Tensor] = {}
for shard_name in shards:
shard_path = f"{checkpoint_path}/{shard_name}"
with safe_open(shard_path, framework="pt", device="cpu") as f:
for key in f.keys(): # noqa: SIM118
if key.startswith(expert_prefix):
all_tensors[key] = f.get_tensor(key)
expert_indices = sorted(
{
int(key.split(".")[idx_pos])
for key in all_tensors
if key.endswith(".gate_proj.weight")
}
)
if max_experts is not None:
expert_indices = expert_indices[:max_experts]
num_experts = len(expert_indices)
gate_weights, up_weights, down_weights = [], [], []
gate_scales, up_scales, down_scales = [], [], []
gate_gscales, up_gscales, down_gscales = [], [], []
a1_scales, a2_scales = [], []
for idx in expert_indices:
prefix = f"{expert_prefix}{idx}"
gate_weights.append(all_tensors[f"{prefix}.gate_proj.weight"])
gate_scales.append(all_tensors[f"{prefix}.gate_proj.weight_scale"])
gate_gscales.append(all_tensors[f"{prefix}.gate_proj.weight_scale_2"])
up_weights.append(all_tensors[f"{prefix}.up_proj.weight"])
up_scales.append(all_tensors[f"{prefix}.up_proj.weight_scale"])
up_gscales.append(all_tensors[f"{prefix}.up_proj.weight_scale_2"])
down_weights.append(all_tensors[f"{prefix}.down_proj.weight"])
down_scales.append(all_tensors[f"{prefix}.down_proj.weight_scale"])
down_gscales.append(all_tensors[f"{prefix}.down_proj.weight_scale_2"])
a1_scales.append(all_tensors[f"{prefix}.gate_proj.input_scale"])
a2_scales.append(all_tensors[f"{prefix}.down_proj.input_scale"])
# Stack into MoE format.
# w1 = [E, 2*intermediate, hidden//2] (gate + up concatenated)
w1 = torch.stack(
[torch.cat([g, u], dim=0) for g, u in zip(gate_weights, up_weights)]
).cuda()
w1_scale = torch.stack(
[torch.cat([g, u], dim=0) for g, u in zip(gate_scales, up_scales)]
).cuda()
w1_gscale = torch.stack(gate_gscales).cuda()
# w2 = [E, hidden, intermediate//2]
w2 = torch.stack(down_weights).cuda()
w2_scale = torch.stack(down_scales).cuda()
w2_gscale = torch.stack(down_gscales).cuda()
a13_scale_raw = torch.stack(a1_scales).cuda()
a2_scale_raw = torch.stack(a2_scales).cuda()
# Apply EMULATION transforms (matches oracle/nvfp4.py).
nvfp4_emulation_utils.kE2M1ToFloat_handle.val = (
nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda()
)
a1_gscale = 1.0 / a13_scale_raw.max().to(torch.float32)
a2_gscale = 1.0 / a2_scale_raw.max().to(torch.float32)
# ── Simulate TP sharding ──
# w1 (gate_up): column-parallel → shard the N dimension (dim 1).
# w2 (down): row-parallel → shard the K dimension (dim 2,
# which is the packed K//2 dim).
# Scales follow the same sharding on the corresponding dimension.
tp = tensor_parallel_size
if tp > 1:
n1 = w1.size(1) // tp
w1 = w1[:, :n1, :].contiguous()
w1_scale = w1_scale[:, :n1, :].contiguous()
k2_packed = w2.size(2) // tp
k2_scale = w2_scale.size(2) // tp
w2 = w2[:, :, :k2_packed].contiguous()
w2_scale = w2_scale[:, :, :k2_scale].contiguous()
hidden_dim = w1.size(2) * 2
intermediate_size = w1.size(1) // 2
return (
w1,
w1_scale,
w1_gscale,
w2,
w2_scale,
w2_gscale,
a1_gscale,
a2_gscale,
num_experts,
hidden_dim,
intermediate_size,
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Triton NVFP4 kernel requires CUDA.",
)
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 1024])
@pytest.mark.parametrize("top_k", [4])
@pytest.mark.parametrize("model_id", list(MOE_MODEL_CONFIGS.keys()))
@pytest.mark.parametrize(
"tensor_parallel_size",
[pytest.param(val, id=f"tensor_parallel_size:{val}") for val in [1, 2, 4, 8]],
)
def test_nvfp4_moe_correctness(
loaded_model_files,
num_tokens: int,
top_k: int,
model_id: str,
tensor_parallel_size: int,
) -> None:
"""Compare Nvfp4QuantizationEmulationTritonExperts (fused weight dequant + compute)
against the unfused reference Nvfp4QuantizationEmulationTritonExpertsReference.
Both must produce bit-identical results.
"""
num_test_experts = max(8, top_k)
(
w1,
w1_scale,
w1_gscale,
w2,
w2_scale,
w2_gscale,
a1_gscale,
a2_gscale,
num_experts,
hidden_dim,
intermediate_size,
) = _load_nvfp4_moe_weights(
loaded_model_files,
model_id,
tensor_parallel_size,
max_experts=num_test_experts,
)
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=top_k,
hidden_dim=hidden_dim,
intermediate_size=intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.TopK,
max_num_tokens=512,
)
def _make_quant_config():
return nvfp4_moe_quant_config(
g1_alphas=w1_gscale.clone(),
g2_alphas=w2_gscale.clone(),
a1_gscale=a1_gscale.clone(),
a2_gscale=a2_gscale.clone(),
w1_scale=w1_scale.clone(),
w2_scale=w2_scale.clone(),
)
ref_experts = Nvfp4QuantizationEmulationTritonExpertsReference(
moe_config=moe_config,
quant_config=_make_quant_config(),
)
fused_experts = Nvfp4QuantizationEmulationTritonExperts(
moe_config=moe_config,
quant_config=_make_quant_config(),
)
torch.manual_seed(42)
hidden_states = torch.randn(
num_tokens, hidden_dim, dtype=torch.bfloat16, device="cuda"
)
topk_weights = torch.randn(
num_tokens, top_k, dtype=torch.float32, device="cuda"
).softmax(dim=-1)
topk_ids = torch.stack(
[torch.randperm(num_experts, device="cuda")[:top_k] for _ in range(num_tokens)]
).to(torch.int32)
N = w1.size(1) # 2 * intermediate
K = hidden_dim
ws13_size = num_tokens * top_k * max(intermediate_size, K)
ws2_size = num_tokens * top_k * max(N, K)
workspace13_ref = torch.zeros(ws13_size, dtype=torch.bfloat16, device="cuda")
workspace2_ref = torch.zeros(ws2_size, dtype=torch.bfloat16, device="cuda")
output_ref = torch.zeros(num_tokens, K, dtype=torch.bfloat16, device="cuda")
workspace13_fused = torch.zeros_like(workspace13_ref)
workspace2_fused = torch.zeros_like(workspace2_ref)
output_fused = torch.zeros_like(output_ref)
apply_kwargs = dict(
hidden_states=hidden_states,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
a1q_scale=None,
a2_scale=None,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
# Unfused reference.
ref_experts.apply(
output=output_ref,
workspace13=workspace13_ref,
workspace2=workspace2_ref,
**apply_kwargs,
)
# Fused implementation.
fused_experts.apply(
output=output_fused,
workspace13=workspace13_fused,
workspace2=workspace2_fused,
**apply_kwargs,
)
# Not strict equality on H100, MI325, MI300 (< 0.1% elements).
# The fused on-the-fly dequant path can lower to a slightly
# different Triton/MMA tiling than the pre-dequantized
# reference; experiments with reference-like tiling/masking
# reduced some diffs were not kept because they regress
# the fused kernel speed.
# Strict equality validated on MI355.
torch.testing.assert_close(
output_fused,
output_ref,
atol=0.0 if on_gfx950() else 0.02,
rtol=0,
)
def _dense_nvfp4_reference(
layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
"""Decode E2M1 nibbles and both scale levels without production helpers."""
levels = kE2M1ToFloat.cpu()
weight = break_fp4_bytes(layer.weight.cpu(), torch.float32)
weight *= (
layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu()
).repeat_interleave(16, dim=-1)
blocks = x.cpu().float().reshape(x.shape[0], -1, 16)
global_scale = layer.input_global_scale_inv.cpu()
scales = (
(blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale)
.clamp(max=FLOAT8_E4M3_MAX)
.to(torch.float8_e4m3fn)
.float()
)
inverse = torch.where(scales == 0, 0, global_scale / scales)
scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX)
distances = (scaled.unsqueeze(-1) - levels).abs()
# E2M1 ties round to the even significand (even encoding).
nearest_mask = distances == distances.amin(-1, keepdim=True)
even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0)
candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask)
nearest = candidates.to(torch.int8).argmax(-1)
activation = levels[nearest] * blocks.sign() * (scales / global_scale)
expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T
if bias is not None:
expected += bias.cpu()
return expected
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)])
@pytest.mark.parametrize("with_bias", [False, True])
def test_nvfp4_dense_emulation(
dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool
) -> None:
"""W4A4 fallback preserves packed scales, bias and outputs across replay."""
set_random_seed(47)
layer = torch.nn.Module()
# Every signed E2M1 code appears, including both zero encodings.
codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k)
layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8)
layer.weight_scale = (
torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8
).to(torch.float8_e4m3fn)
layer.weight_global_scale = torch.tensor(0.25, device="cuda")
layer.input_global_scale_inv = torch.tensor(3.0, device="cuda")
x = torch.randn(m, k, device="cuda", dtype=dtype)
x[:, :16] = 0
bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None
with set_current_vllm_config(
VllmConfig(kernel_config=KernelConfig(linear_backend="emulation"))
):
kernel = init_nvfp4_linear_kernel()
assert isinstance(kernel, EmulationNvFp4LinearKernel)
kernel.process_weights_after_loading(layer)
original_weight = layer.weight.clone()
original_scale = layer.weight_scale.view(torch.uint8).clone()
actual = kernel.apply_weights(layer, x, bias)
torch.testing.assert_close(
actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
)
assert actual.abs().max() > 0.1
saved = actual.clone()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = kernel.apply_weights(layer, x, bias)
for sign in (-1, 1):
x.mul_(sign * 0.75)
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(
captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
)
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
assert torch.equal(layer.weight, original_weight)
assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale)
@@ -1,32 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for NVFP4 linear kernel selection order (CPU-only)."""
import pytest
from vllm.model_executor.kernels.linear import (
_POSSIBLE_NVFP4_KERNELS,
CutlassNvFp4LinearKernel,
FlashInferB12xNvFp4LinearKernel,
FlashInferCuteDslNvFp4W4A16LinearKernel,
FlashInferCutlassNvFp4LinearKernel,
)
from vllm.platforms.interface import PlatformEnum
# W4A4 kernels that run on SM120/121, where the head of the list is gated to
# sm_10x and selection falls through to whatever follows.
W4A4_KERNELS_ON_SM12X = (
FlashInferCutlassNvFp4LinearKernel,
FlashInferB12xNvFp4LinearKernel,
CutlassNvFp4LinearKernel,
)
@pytest.mark.parametrize("w4a4_kernel", W4A4_KERNELS_ON_SM12X)
def test_w4a16_kernel_does_not_precede_w4a4_kernels(w4a4_kernel):
candidates = _POSSIBLE_NVFP4_KERNELS[PlatformEnum.CUDA]
w4a16_index = candidates.index(FlashInferCuteDslNvFp4W4A16LinearKernel)
assert candidates.index(w4a4_kernel) < w4a16_index, (
f"{w4a4_kernel.__name__} must be preferred over "
f"{FlashInferCuteDslNvFp4W4A16LinearKernel.__name__}"
)
@@ -1,346 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
reason="Nvfp4 Requires compute capability of 10 or above.",
allow_module_level=True,
)
DTYPES = [torch.float16, torch.bfloat16]
SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)]
PAD_SHAPES = [
(90, 64),
(150, 64),
(128, 48),
(128, 80),
(150, 80),
(90, 48),
(90, 128),
(150, 128),
(150, 48),
(90, 80),
(128, 512),
(128, 1024),
(128, 2048),
(64, 7168),
(64, 7152),
(32, 14336),
]
PADDED_OUTPUT_SHAPES = [(128, 48), (128, 80), (150, 48), (150, 80), (64, 7152)]
SEEDS = [42]
CUDA_DEVICES = ["cuda:0"]
FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
# E2M1 to float
# 0111 -> 6
# 0110 -> 4
# 0101 -> 3
# 0100 -> 2
# 0011 -> 1.5
# 0010 -> 1
# 0001 -> 0.5
# 0000 -> 0
E2M1_TO_FLOAT32 = [
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
]
BLOCK_SIZE = 16
def cast_from_fp4(x, m, n):
# The fp4 values are packed in uint8 as [v_1st | v_2nd]
v_2nd = x & 0xF
v_1st = (x >> 4) & 0xF
c = torch.stack((v_2nd, v_1st), dim=-1)
out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()])
out = out.reshape(m, n).to(torch.float32)
return out
def cast_to_fp4(x):
sign = torch.sign(x)
x = torch.abs(x)
x[(x >= 0.0) & (x <= 0.25)] = 0.0
x[(x > 0.25) & (x < 0.75)] = 0.5
x[(x >= 0.75) & (x <= 1.25)] = 1.0
x[(x > 1.25) & (x < 1.75)] = 1.5
x[(x >= 1.75) & (x <= 2.5)] = 2.0
x[(x > 2.5) & (x < 3.5)] = 3.0
x[(x >= 3.5) & (x <= 5.0)] = 4.0
x[x > 5.0] = 6.0
return x * sign
def get_reciprocal(x):
if isinstance(x, torch.Tensor):
return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
elif isinstance(x, (float, int)):
return 0.0 if x == 0 else 1.0 / x
else:
raise TypeError("Input must be a float, int, or a torch.Tensor.")
def ref_nvfp4_quant(x, global_scale):
assert global_scale.dtype == torch.float32
assert x.ndim == 2
m, n = x.shape
x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE))
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
scaled_x = x.to(torch.float32) * output_scale
clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
return cast_to_fp4(clipped_x), scale.squeeze(-1)
def recover_swizzled_scales(scale, m, n, include_padding=False):
round_up = lambda x, y: (x + y - 1) // y * y
rounded_m = round_up(m, 128)
scale_n = n // BLOCK_SIZE
rounded_n = round_up(scale_n, 4)
# Recover the swizzled scaling factor to linear layout
tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
if include_padding:
return result
return result[:m, :scale_n]
def round_up(x: int, y: int) -> int:
return (x + y - 1) // y * y
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_quantize_to_fp4(
dtype: torch.dtype,
shape: tuple[int, int],
seed: int,
device: str,
) -> None:
set_random_seed(seed)
torch.set_default_device(device)
m, n = shape
x = torch.randn((m, n), dtype=dtype)
tensor_amax = torch.abs(x).max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
out, out_scale = ops.scaled_fp4_quant(x, global_scale)
scale_ans = recover_swizzled_scales(out_scale, m, n)
out_ans = cast_from_fp4(out, m, n)
torch.testing.assert_close(out_ans, out_ref)
torch.testing.assert_close(scale_ans, scale_ref)
@pytest.mark.parametrize(
"shape",
[(32, 4096), (128, 4096), (1, 64), (127, 1024), (256, 16384)],
)
@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False])
@torch.inference_mode()
def test_python_util_matches_cpp_allocation(
shape: tuple[int, int],
is_sf_swizzled_layout: bool,
) -> None:
"""Verify that the Python utility (create_fp4_output_tensors) allocates
tensors with the same shapes and dtypes as the C++ functional variant
(scaled_fp4_quant_func).
"""
from vllm._custom_ops import create_fp4_output_tensors
torch.set_default_device("cuda:0")
m, n = shape
input_tensor = torch.randn((m, n), dtype=torch.bfloat16)
input_scale = torch.tensor([1.0], dtype=torch.float32, device="cuda:0")
# C++ functional variant allocates internally
cpp_out, cpp_scale = torch.ops._C.scaled_fp4_quant(
input_tensor, input_scale, is_sf_swizzled_layout
)
# Python utility
py_out, py_scale = create_fp4_output_tensors(
m, n, torch.device("cuda:0"), is_sf_swizzled_layout
)
assert py_out.shape == cpp_out.shape, (
f"Output shape mismatch: Python {py_out.shape} vs C++ {cpp_out.shape}"
)
assert py_out.dtype == cpp_out.dtype, (
f"Output dtype mismatch: Python {py_out.dtype} vs C++ {cpp_out.dtype}"
)
assert py_scale.shape == cpp_scale.shape, (
f"Scale shape mismatch: Python {py_scale.shape} vs C++ {cpp_scale.shape}"
)
assert py_scale.dtype == cpp_scale.dtype, (
f"Scale dtype mismatch: Python {py_scale.dtype} vs C++ {cpp_scale.dtype}"
)
@pytest.mark.parametrize("shape", PADDED_OUTPUT_SHAPES)
@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False])
@torch.inference_mode()
def test_quantize_to_fp4_with_padded_output(
shape: tuple[int, int],
is_sf_swizzled_layout: bool,
) -> None:
from vllm._custom_ops import create_fp4_output_tensors
dtype = torch.float16
set_random_seed(42)
torch.set_default_device("cuda:0")
m, n = shape
padded_n = round_up(n, 32)
assert padded_n > n
x = torch.randn((m, n), dtype=dtype)
tensor_amax = torch.abs(x).max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
out, out_scale = ops.scaled_fp4_quant(
x,
global_scale,
is_sf_swizzled_layout=is_sf_swizzled_layout,
padded_n=padded_n,
)
py_out, py_scale = create_fp4_output_tensors(
m,
n,
torch.device("cuda:0"),
is_sf_swizzled_layout,
padded_n=padded_n,
)
assert out.shape == (m, padded_n // 2)
assert out.shape == py_out.shape
assert out_scale.shape == py_scale.view(torch.float8_e4m3fn).shape
out_ans = cast_from_fp4(out[:, : n // 2], m, n)
torch.testing.assert_close(out_ans, out_ref)
assert torch.count_nonzero(out[:, n // 2 :]) == 0
if is_sf_swizzled_layout:
scale_ans = recover_swizzled_scales(out_scale, m, padded_n)
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
else:
scale_ans = out_scale.to(torch.float32)
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
@pytest.mark.parametrize(
("shape", "padded_n"),
[((90, 80), None), ((90, 48), 64)],
)
@torch.inference_mode()
def test_swizzled_scale_padding_initialized_in_kernel(
shape: tuple[int, int], padded_n: int | None
) -> None:
"""The out variant must overwrite dirty scale padding without a memset."""
torch.set_default_device("cuda:0")
m, n = shape
physical_n = padded_n if padded_n is not None else n
rounded_m = round_up(m, 128)
rounded_scale_n = round_up(physical_n // BLOCK_SIZE, 4)
x = torch.randn((m, n), dtype=torch.float16)
global_scale = torch.tensor(1.0, dtype=torch.float32)
out = torch.full((m, physical_n // 2), 0xFF, dtype=torch.uint8)
out_scale = torch.full(
(rounded_m, rounded_scale_n // 4), 0x7F7F7F7F, dtype=torch.int32
)
torch.ops._C.scaled_fp4_quant.out(
x,
global_scale,
True,
output=out,
output_scale=out_scale,
)
full_scale = recover_swizzled_scales(
out_scale.view(torch.float8_e4m3fn), m, physical_n, include_padding=True
)
assert torch.count_nonzero(full_scale[m:, :]) == 0
assert torch.count_nonzero(full_scale[:m, n // BLOCK_SIZE :]) == 0
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
@torch.inference_mode()
def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
dtype = torch.float16
set_random_seed(42)
torch.set_default_device("cuda:0")
m, n = pad_shape
x = torch.randn((m, n), dtype=dtype)
tensor_amax = torch.abs(x).max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
out, out_scale = ops.scaled_fp4_quant(x, global_scale)
scale_ans = recover_swizzled_scales(out_scale, m, n)
out_ans = cast_from_fp4(out, m, n)
torch.testing.assert_close(out_ans, out_ref)
torch.testing.assert_close(scale_ans, scale_ref)
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
@torch.inference_mode()
def test_quantize_to_fp4_padded_no_sf_swizzled(pad_shape: tuple[int, int]) -> None:
dtype = torch.float16
set_random_seed(42)
torch.set_default_device("cuda:0")
m, n = pad_shape
x = torch.randn((m, n), dtype=dtype)
tensor_amax = torch.abs(x).max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
out, out_scale = ops.scaled_fp4_quant(x, global_scale, is_sf_swizzled_layout=False)
scale_ans = out_scale.to(torch.float32)
out_ans = cast_from_fp4(out, m, n)
torch.testing.assert_close(out_ans, out_ref)
torch.testing.assert_close(scale_ans, scale_ref)
@@ -1,269 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Copyright (C) 2025 Roberto L. Castro (Roberto.LopezCastro@ist.ac.at).
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import numpy as np
import pytest
import torch
from compressed_tensors.transform.utils.hadamard import deterministic_hadamard_matrix
from vllm import _custom_ops as ops # use existing nvfp4 gemm in vllm
from vllm._custom_ops import fusedQuantizeNv
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not torch.cuda.is_available():
pytest.skip("CUDA required for these tests.", allow_module_level=True)
if not (
current_platform.has_device_capability(100)
or current_platform.has_device_capability(120)
):
pytest.skip(
reason="Tests require compute capability 10.0 (100) or 12.0 (120).",
allow_module_level=True,
)
# ----- Helpers -----
def get_hadamard_matrix(group_size: int, dtype: torch.dtype, device: torch.device):
return (
deterministic_hadamard_matrix(group_size, dtype=dtype, device=device)
* group_size**-0.5
)
def _rtne_fp4(x: torch.Tensor):
device = x.device
grid = torch.tensor(
[
-6.0,
-4.0,
-3.0,
-2.0,
-1.5,
-1.0,
-0.5,
-0.0,
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
],
dtype=x.dtype,
device=x.device,
)
grid_int = torch.tensor(
[-1, -2, -3, -4, -5, -6, -7, -8, 0, 1, 2, 3, 4, 5, 6, 7],
dtype=torch.uint8,
device=device,
)
inds = torch.bucketize(x, grid)
lo, hi = (inds - 1).clamp(min=0, max=15), inds.clamp(min=0, max=15)
g_lo, g_hi = grid[lo], grid[hi]
pick_hi = (g_hi - x < x - g_lo) | (g_hi - x == x - g_lo) & (grid_int[hi] % 2 == 0)
y = torch.where(pick_hi, g_hi, g_lo)
y_int = torch.where(pick_hi, grid_int[hi], grid_int[lo])
y_int_packed = (y_int[..., 1::2] & 0xF) << 4 | y_int[..., ::2] & 0xF
return y, y_int_packed
def _dq_fp4(x_e2m1: torch.Tensor, x_e4m3: torch.Tensor, alpha: float):
device = x_e2m1.device
x_e2m1_i32 = x_e2m1.view(dtype=torch.uint8).to(dtype=torch.int32)
x_e2m1_unpacked = torch.stack(
[x_e2m1_i32 & 0xF, (x_e2m1_i32 >> 4) & 0xF], dim=-1
).flatten(start_dim=-2)
grid_dq = torch.tensor(
[
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
],
dtype=torch.float64,
device=device,
)
x_fp4_dq = grid_dq[x_e2m1_unpacked]
scales_dq = x_e4m3.to(torch.float64)
x_dq = (x_fp4_dq.unflatten(dim=-1, sizes=(-1, 16)) * scales_dq[..., None]).flatten(
start_dim=-2
) / alpha # * (4. / 3.)
return x_dq, x_fp4_dq, scales_dq
def _unpack_mask(clip_mask: torch.Tensor) -> torch.Tensor:
clip_mask_unpacked_dq = torch.zeros(
*clip_mask.shape[:-1],
clip_mask.size(-1) * 8,
dtype=torch.bool,
device=clip_mask.device,
)
for i in range(8):
clip_mask_unpacked_dq[..., i::8] = (clip_mask >> i) & 1
return clip_mask_unpacked_dq
def _forward_quantize_ref(x: torch.Tensor, h: torch.Tensor, rot_size: int):
device = x.device
xh_ref64 = (
x.unflatten(dim=-1, sizes=(-1, rot_size)).to(dtype=torch.float64)
@ h.reshape(rot_size, rot_size).to(dtype=torch.float64)
).flatten(start_dim=-2)
abs_max = xh_ref64.unflatten(dim=-1, sizes=(-1, 16)).abs().amax(dim=-1)
scales_ref64_ = abs_max + 1e-8
xh_e4m3_ref = scales_ref64_.to(dtype=torch.float8_e4m3fn)
scales_ref64 = xh_e4m3_ref.to(dtype=torch.float64)
xh_scaled_ref64 = (
xh_ref64.unflatten(dim=-1, sizes=(-1, 16)) / scales_ref64[..., None]
).flatten(start_dim=-2)
xh_scaled_ref64 *= 6.0
clip_mask_unpacked_ref = xh_scaled_ref64.abs() < 6.0
clip_mask_ref = torch.zeros(
*x.shape[:-1], x.size(-1) // 8, dtype=torch.uint8, device=device
)
for i in range(8):
clip_mask_ref |= clip_mask_unpacked_ref[..., i::8].to(dtype=torch.uint8) << i
xh_fp4_ref, xh_e2m1_ref = _rtne_fp4(xh_scaled_ref64)
xh_dq, xh_fp4_dq, scales_dq = _dq_fp4(xh_e2m1_ref, xh_e4m3_ref, 6.0)
clip_mask_unpacked_dq = _unpack_mask(clip_mask_ref)
assert xh_fp4_dq.equal(xh_fp4_ref)
assert scales_dq.equal(scales_ref64)
assert clip_mask_unpacked_dq.equal(clip_mask_unpacked_ref)
return (
xh_dq,
clip_mask_unpacked_ref,
(xh_e2m1_ref, xh_e4m3_ref, clip_mask_ref),
)
DTYPE = torch.bfloat16
DEVICE = torch.device("cuda:0")
ROT_SIZES = [16, 32, 64, 128]
GLOBAL_SCALES = [6.0]
LLAMA_MODELS = {
"7B": [(4096, 3 * 4096), (4096, 4096), (4096, 2 * 10752), (10752, 4096)],
"13B": [(5120, 3 * 5120), (5120, 5120), (5120, 2 * 13568), (13568, 5120)],
"33B": [(6656, 3 * 6656), (6656, 6656), (6656, 2 * 17664), (17664, 6656)],
"70B": [(8192, 3 * 8192), (8192, 8192), (8192, 2 * 21760), (21760, 8192)],
}
@pytest.fixture(autouse=True)
def _seed_each_test():
set_random_seed(0)
np.random.seed(0)
torch.random.manual_seed(0)
@pytest.mark.parametrize("rot_size", ROT_SIZES)
@pytest.mark.parametrize("global_scale_value", GLOBAL_SCALES)
@torch.inference_mode()
def test_fused_quantization(rot_size: int, global_scale_value: float):
dtype, device = DTYPE, DEVICE
h = get_hadamard_matrix(rot_size, dtype, device)
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
global_scale = torch.tensor([global_scale_value], device=device)
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size)
xh_e2m1, xh_e4m3 = fusedQuantizeNv(x, h, global_scale)
xh_e4m3 = xh_e4m3.reshape(2, 4096, 4096 // 16)
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e4m3, alpha=global_scale_value)
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-1
m, n, k = 504, 4096 * 2, 4096
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
a_e2m1, a_e4m3 = fusedQuantizeNv(a, h, global_scale)
b_e2m1, b_e4m3 = fusedQuantizeNv(b, h, global_scale)
a_dq, *_ = _dq_fp4(a_e2m1, a_e4m3[:m, :k], alpha=1.0)
b_dq, *_ = _dq_fp4(b_e2m1, b_e4m3[:n, :k], alpha=1.0)
out_ref = a_dq @ b_dq.transpose(-2, -1)
a_scale_block = to_blocked(a_e4m3, backend="triton").view(-1, k // 16)
b_scale_block = to_blocked(b_e4m3, backend="triton").view(-1, k // 16)
alpha = torch.tensor([1.0], device=device)
out = ops.cutlass_scaled_fp4_mm(
a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha, torch.bfloat16
)
assert out.equal(out_ref.to(dtype=out.dtype))
@pytest.mark.parametrize("model", list(LLAMA_MODELS.keys()))
@pytest.mark.parametrize("layer_idx", [0, 1, 2, 3])
@pytest.mark.parametrize("batch", [1, 16])
@pytest.mark.parametrize("rot_size", ROT_SIZES)
@torch.inference_mode()
def test_llama_shapes(model: str, layer_idx: int, batch: int, rot_size: int):
dtype, device = DTYPE, DEVICE
m = batch
k, n = LLAMA_MODELS[model][layer_idx]
h = get_hadamard_matrix(rot_size, dtype, device)
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
global_scale = torch.tensor([1.0], device=device)
a_e2m1, a_e4m3 = fusedQuantizeNv(a, h, global_scale)
b_e2m1, b_e4m3 = fusedQuantizeNv(b, h, global_scale)
a_dq, *_ = _dq_fp4(a_e2m1, a_e4m3[:m, :k], alpha=1.0)
b_dq, *_ = _dq_fp4(b_e2m1, b_e4m3[:n, :k], alpha=1.0)
out_ref = a_dq @ b_dq.transpose(-2, -1)
a_scale_block = to_blocked(a_e4m3, backend="triton").view(-1, k // 16)
b_scale_block = to_blocked(b_e4m3, backend="triton").view(-1, k // 16)
alpha = torch.tensor([1.0], device=device)
out = ops.cutlass_scaled_fp4_mm(
a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha, torch.bfloat16
)
assert out.equal(out_ref.to(dtype=out.dtype))
@@ -1,100 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, dequantize_nvfp4_to_dtype
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
reason="Nvfp4 Requires compute capability of 10 or above.",
allow_module_level=True,
)
DTYPES = [torch.float16, torch.bfloat16]
# m, n, k
SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)]
PAD_SHAPES = [(150, 128, 64), (128, 128, 96)]
SHAPES.extend(PAD_SHAPES)
SEEDS = [42]
CUDA_DEVICES = ["cuda:0"]
def get_ref_results(
a_fp4,
b_fp4,
a_sf,
b_sf,
a_global_scale,
b_global_scale,
m,
n,
dtype,
block_size,
device,
):
_, m_k = a_fp4.shape
_, n_k = b_fp4.shape
assert m_k == n_k
a_in_dtype = dequantize_nvfp4_to_dtype(
a_fp4, a_sf, a_global_scale, dtype=dtype, device=device, block_size=block_size
)
b_in_dtype = dequantize_nvfp4_to_dtype(
b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size
)
return torch.matmul(a_in_dtype, b_in_dtype.t())
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("shape", SHAPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@torch.inference_mode()
def test_nvfp4_gemm(
dtype: torch.dtype,
shape: tuple[int, int, int],
seed: int,
device: str,
) -> None:
set_random_seed(seed)
m, n, packed_k = shape
k = packed_k * 2
block_size = 16
a_dtype = torch.randn((m, k), dtype=dtype, device=device)
b_dtype = torch.randn((n, k), dtype=dtype, device=device)
a_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1)
).to(torch.float32)
b_global_scale = (
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1)
).to(torch.float32)
alpha = 1.0 / (a_global_scale * b_global_scale)
# ops.scaled_fp4_quant returns swizzled scales, while weights
# from checkpoints are in linear scales.
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a_dtype, a_global_scale)
b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(b_dtype, b_global_scale)
# get_ref_results unswizzles the scales internally.
expected_out = get_ref_results(
a_fp4,
b_fp4,
a_scale_interleaved,
b_scale_interleaved,
a_global_scale,
b_global_scale,
m,
n,
dtype,
block_size,
device,
)
out = ops.cutlass_scaled_fp4_mm(
a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype
)
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
@@ -1,541 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import patch
import pytest
import torch
from vllm.model_executor.layers.quantization.utils import fp8_utils, int8_utils
from vllm.model_executor.layers.quantization.utils.quant_utils import get_fp8_min_max
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"shape",
[
(0, 128),
(1, 128),
(3, 384),
(5, 384),
(7, 384),
(15, 384),
(31, 128),
(32, 128),
(63, 256),
(64, 256),
(16, 512),
(3, 2688),
(3, 5376),
(333, 128),
(512, 6144),
],
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("column_major", [False, True])
@pytest.mark.parametrize("tma_aligned", [False, True])
@pytest.mark.parametrize("scale_ue8m0", [False, True])
@pytest.mark.parametrize("group_size", [64, 128])
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on CUDA/ROCm/XPU.",
)
def test_per_token_group_quant_fp8(
shape,
dtype: torch.dtype,
column_major: bool,
tma_aligned: bool,
scale_ue8m0: bool,
group_size: int,
):
"""Tail groups preserve FP8 values and scales in every supported layout."""
device = current_platform.device_type
torch.manual_seed(42)
num_tokens, hidden_dim = shape
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=dtype) * 8
# native kernel path
out_q, scale = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
column_major_scales=column_major,
tma_aligned_scales=tma_aligned,
use_ue8m0=scale_ue8m0,
)
# triton ref
with (
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False),
patch("vllm.platforms.current_platform.is_xpu", return_value=False),
):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
column_major_scales=column_major,
use_ue8m0=scale_ue8m0,
)
assert torch.allclose(out_q.float(), ref_q.float(), atol=0.15, rtol=0.15)
assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01)
def _misaligned_clone(x: torch.Tensor) -> torch.Tensor:
"""Contiguous copy of ``x`` whose ``data_ptr`` is not 16B aligned."""
numel = x.numel()
base = torch.empty(numel + 16 // x.element_size(), device=x.device, dtype=x.dtype)
view = base[1 : 1 + numel]
if view.data_ptr() % 16 == 0:
pytest.skip("allocator did not yield a misaligned offset")
return view.view(x.shape).copy_(x)
@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)])
@pytest.mark.parametrize("column_major", [False, True])
@pytest.mark.parametrize("scale_ue8m0", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="register-resident schedule is CUDA/ROCm only",
)
def test_per_token_group_quant_fp8_register_matches_smem(
shape, column_major: bool, scale_ue8m0: bool, dtype: torch.dtype
):
"""group_size=128 with 16-bit input takes a register-resident schedule that
needs 16B-aligned buffers; a misaligned input falls back to the
shared-memory schedule. The two must agree bit-for-bit, which the
tolerance-based comparison against Triton above cannot establish.
"""
device = current_platform.device_type
torch.manual_seed(42)
x = torch.randn(shape, device=device, dtype=dtype) * 8
reg_q, reg_s = fp8_utils.per_token_group_quant_fp8(
x, 128, column_major_scales=column_major, use_ue8m0=scale_ue8m0
)
smem_q, smem_s = fp8_utils.per_token_group_quant_fp8(
_misaligned_clone(x),
128,
column_major_scales=column_major,
use_ue8m0=scale_ue8m0,
)
assert torch.equal(reg_q.view(torch.uint8), smem_q.view(torch.uint8))
assert torch.equal(reg_s, smem_s)
@pytest.mark.parametrize(
"num_tokens,hidden_dim,group_size",
[
# No padding: mn=4 (mult of 4), groups_per_row=56 (mult of 4)
(4, 7168, 128),
# MN padding only: mn=1, tma_aligned_mn=4
(1, 7168, 128),
# MN padding only: mn=3, tma_aligned_mn=4
(3, 7168, 128),
# K padding only: groups_per_row=5 (5%4=1)
(4, 640, 128),
# K padding only: groups_per_row=6 (6%4=2)
(4, 768, 128),
# Single packed column, no padding: k_num_packed=1, mn%4=0
(4, 384, 128),
# Both MN and K padding
(1, 384, 128),
(3, 640, 128),
# Larger shapes with no padding
(64, 7168, 128),
(128, 14336, 128),
# Larger shapes with padding
(127, 7168, 128),
(253, 640, 128),
(1, 32, 32),
(3, 64, 32),
(7, 160, 32),
(32, 512, 32),
(127, 4096, 32),
],
)
@pytest.mark.parametrize("poisoned_scales", [False, True])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="DeepGEMM not available on this platform",
)
def test_per_token_group_quant_fp8_packed(
num_tokens, hidden_dim, group_size, poisoned_scales
):
"""Test the packed DeepGEMM quantization kernel against the Triton
reference (row-major, UE8M0 scales)."""
device = "cuda"
torch.manual_seed(42)
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
mn = num_tokens
groups_per_row = hidden_dim // group_size
k_num_packed = (groups_per_row + 3) // 4
tma_aligned_mn = ((mn + 3) // 4) * 4
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
if poisoned_scales:
# Call the kernel with poisoned scale buffer to
# ensure padded indices are correctly zeroed.
fp8_dtype = current_platform.fp8_dtype()
fp8_min, fp8_max = get_fp8_min_max()
out_q = torch.empty_like(x, dtype=fp8_dtype)
out_s_packed = torch.empty_strided(
(mn, k_num_packed),
(1, tma_aligned_mn),
device=device,
dtype=torch.int32,
)
torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).fill_(0x7F7F7F7F)
torch.ops._C.per_token_group_fp8_quant_packed(
x,
out_q,
out_s_packed,
group_size,
1e-10,
fp8_min,
fp8_max,
)
else:
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
x,
group_size=group_size,
use_ue8m0=True,
)
# Triton reference (row-major float32 scales, UE8M0)
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
use_ue8m0=True,
)
# Quantized values must match.
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
# Verify packed scales (valid exponents + padding zeros).
ref_s_flat = ref_s.reshape(mn, groups_per_row)
ref_exponents = (ref_s_flat.view(torch.int32) >> 23) & 0xFF
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
for row in range(mn):
for g in range(groups_per_row):
pack_col = g // 4
pos = g % 4
idx = pack_col * tma_aligned_mn + row
expected[idx] |= int(ref_exponents[row, g].item()) << (pos * 8)
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
assert torch.equal(actual, expected), (
f"Packed scale storage mismatch.\n"
f"First diff at index "
f"{(actual != expected).nonzero(as_tuple=True)[0][0].item()}"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="DeepGEMM not available on this platform",
)
def test_per_token_group_quant_fp8_packed_all_zero():
"""All-zero input must produce well-defined UE8M0 scale bytes via the eps
floor in the kernel's UE8M0 path. Locks down the all-zero behavior before
optimization.
The CUDA kernel computes:
y_s = eps / fp8_max
y_s = exp2(ceil(log2(fmax(y_s, 1e-10))))
For all-zero input, eps/fp8_max < 1e-10, so the inner fmax clamps back to
1e-10, giving exp2(ceil(log2(1e-10))) = exp2(-33) => UE8M0 byte 0x5E (94).
"""
device = "cuda"
num_tokens, hidden_dim, group_size = 4, 7168, 128
x = torch.zeros((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16)
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
x,
group_size=group_size,
use_ue8m0=True,
)
# Quantized values must be all zero.
assert torch.equal(
out_q.view(torch.uint8),
torch.zeros_like(out_q, dtype=torch.uint8),
), "All-zero input should produce all-zero FP8 output"
# UE8M0 byte produced by the kernel for all-zero input.
# The kernel's inner fmax(y_s, 1e-10) clamps eps/fp8_max back to 1e-10.
# 1e-10 as float32 has biased exponent 0x5D and a non-zero mantissa, so
# the kernel's bit-twiddle (exp_bits + (mant_bits != 0)) rounds up to
# 0x5E. This matches exp2(ceil(log2(1e-10))) = exp2(-33).
expected_exp_byte = 0x5E
mn = num_tokens
groups_per_row = hidden_dim // group_size
k_num_packed = (groups_per_row + 3) // 4
tma_aligned_mn = ((mn + 3) // 4) * 4
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
# All valid scale slots must contain the expected packed value.
# Padding slots must be zero.
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
for row in range(mn):
for g in range(groups_per_row):
pack_col = g // 4
pos = g % 4
idx = pack_col * tma_aligned_mn + row
expected[idx] |= expected_exp_byte << (pos * 8)
assert torch.equal(actual, expected), "All-zero scale bytes mismatch"
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="DeepGEMM not available on this platform",
)
def test_per_token_group_quant_fp8_packed_mantissa_rounds_up():
"""Inputs whose absmax/max_8bit produces a non-power-of-2 force the
mantissa-rounding-up branch (exp_byte += 1). Locks down this behavior
before optimization."""
device = "cuda"
num_tokens, hidden_dim, group_size = 4, 7168, 128
# Build a tensor whose per-group absmax = 1.5 * fp8_max * 2^k for various k.
# fp8_max = torch.finfo(torch.float8_e4m3fn).max = 448.0.
# Then absmax/fp8_max = 1.5 * 2^k -> non-zero mantissa, triggers ceil
# rounding to 2^(k+1). Use k=0 for simplicity; the bf16 representation of
# 1.5*448=672.0 is exact.
x = torch.full(
(num_tokens, hidden_dim),
672.0,
device=device,
dtype=torch.bfloat16,
)
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
x,
group_size=group_size,
use_ue8m0=True,
)
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x,
group_size,
use_ue8m0=True,
)
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
mn = num_tokens
groups_per_row = hidden_dim // group_size
k_num_packed = (groups_per_row + 3) // 4
tma_aligned_mn = ((mn + 3) // 4) * 4
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
ref_s_flat = ref_s.reshape(mn, groups_per_row)
ref_exponents = (ref_s_flat.view(torch.int32) >> 23) & 0xFF
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
for row in range(mn):
for g in range(groups_per_row):
pack_col = g // 4
pos = g % 4
idx = pack_col * tma_aligned_mn + row
expected[idx] |= int(ref_exponents[row, g].item()) << (pos * 8)
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
assert torch.equal(actual, expected), "Scale bytes mismatch"
@pytest.mark.parametrize(
"num_tokens,hidden_dim",
[
(1, 7168), # mn padded 1 -> 4
(2, 7168), # mn padded 2 -> 4
(3, 7168), # mn padded 3 -> 4
(5, 7168), # mn padded 5 -> 8
(127, 7168), # mn padded 127 -> 128
(253, 640), # both mn and groups padded
(1, 384), # extreme: 1 group, 1 mn row -> both axes padded
],
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="DeepGEMM not available on this platform",
)
def test_per_token_group_quant_fp8_packed_zero_fills_padded_output_q(
num_tokens, hidden_dim
):
"""When output_q is allocated with shape (tma_aligned_mn, k) instead of
(mn, k), the kernel must overwrite the padded mn rows with zeros so
callers can use ``torch.empty`` instead of ``torch.zeros``."""
device = "cuda"
group_size = 128
torch.manual_seed(42)
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
mn = num_tokens
groups_per_row = hidden_dim // group_size
k_num_packed = (groups_per_row + 3) // 4
tma_aligned_mn = ((mn + 3) // 4) * 4
fp8_dtype = current_platform.fp8_dtype()
fp8_min, fp8_max = get_fp8_min_max()
# Allocate output_q with the padded mn extent and pre-fill with 0xFF
# so the kernel cannot rely on a clean buffer.
out_q = torch.empty((tma_aligned_mn, hidden_dim), device=device, dtype=fp8_dtype)
out_q.view(torch.uint8).fill_(0xFF)
out_s_packed = torch.empty_strided(
(mn, k_num_packed),
(1, tma_aligned_mn),
device=device,
dtype=torch.int32,
)
torch.ops._C.per_token_group_fp8_quant_packed(
x, out_q, out_s_packed, group_size, 1e-10, fp8_min, fp8_max
)
# Live rows must match the Triton reference.
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
ref_q, _ = fp8_utils.per_token_group_quant_fp8(x, group_size, use_ue8m0=True)
assert torch.equal(out_q[:mn], ref_q), "Live region mismatch"
# Padded rows must be all-zero; without this, downstream TMA loads would
# see uninitialised data.
if tma_aligned_mn > mn:
padded_bytes = out_q[mn:tma_aligned_mn].view(torch.uint8)
assert padded_bytes.eq(0).all(), (
f"Padded rows [{mn}, {tma_aligned_mn}) not zeroed; "
f"{padded_bytes.ne(0).sum().item()} non-zero bytes"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="packed FP8 per-token-group quant kernel requires a CUDA-alike GPU",
)
def test_per_token_group_quant_fp8_packed_large_mn():
"""Regression test for https://github.com/vllm-project/vllm/issues/45099.
Some background: gridDim.x and gridDim.y have different limits of 2^31 - 1 and
2^16 - 1, respectively.
Prior code introduced a bug where it incorrectly assumed grid.x and y both have
2^31 - 1 limits and mixed them up, which doesn't surface until the kernel is
launched with a large mn that exceeds grid.y limit (2^16 - 1).
This issue doesn't surface often because each forward pass only processes a
bounded token batch, not the full context.
Quantizing tensors with more rows than that will fail at launch with
"CUDA error: invalid argument".
This is a differential test that compares fp8 output against Triton output
reference when token size sits just above the gridDim.y 2^16 - 1 limit.
"""
device = "cuda"
group_size = 128
# hidden 2048 -> 2048/128 = 16 groups per row -> kx=16, ry=1: one grid row per mn
# row, so any mn > 65535 overflowed grid.y before the fix.
num_tokens, hidden_dim = 65537, 2048
torch.manual_seed(42)
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
x,
group_size=group_size,
use_ue8m0=True,
)
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
x, group_size, use_ue8m0=True
)
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
# Vectorized packed-scale check; the per-element loop used by the smaller
# tests is too slow at this size. groups_per_row is a multiple of 4 here,
# so there is no K padding and the packed view lines up.
mn = num_tokens
groups_per_row = hidden_dim // group_size
k_num_packed = (groups_per_row + 3) // 4
assert groups_per_row % 4 == 0
ref_exponents = (ref_s.reshape(mn, groups_per_row).view(torch.int32) >> 23) & 0xFF
exp = ref_exponents.view(mn, k_num_packed, 4)
expected = (
exp[..., 0] | (exp[..., 1] << 8) | (exp[..., 2] << 16) | (exp[..., 3] << 24)
)
assert torch.equal(out_s_packed.cpu(), expected.cpu()), "Packed scale mismatch"
@pytest.mark.parametrize(
"shape",
[
(0, 128),
(1, 128),
(3, 384),
(5, 384),
(7, 384),
(15, 384),
(32, 128),
(64, 256),
(16, 512),
],
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("group_size", [64, 128])
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on CUDA/ROCm/XPU.",
)
def test_per_token_group_quant_int8(shape, dtype, group_size: int):
"""The shared launch policy also handles INT8 tail groups."""
device = current_platform.device_type
torch.manual_seed(42)
num_tokens, hidden_dim = shape
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=dtype) * 8
# cuda path
out_q, scale = int8_utils.per_token_group_quant_int8(
x,
group_size,
)
# triton ref
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
ref_q, ref_s = int8_utils.per_token_group_quant_int8(
x,
group_size,
)
assert torch.allclose(out_q.float(), ref_q.float(), atol=0.15, rtol=0.15)
assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01)
@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="register-resident schedule is CUDA/ROCm only",
)
def test_per_token_group_quant_int8_register_matches_smem(shape, dtype: torch.dtype):
"""int8 output shares the register-resident schedule with fp8, so it needs
the same bit-exactness guard against the shared-memory schedule.
"""
device = current_platform.device_type
torch.manual_seed(42)
x = torch.randn(shape, device=device, dtype=dtype) * 8
reg_q, reg_s = int8_utils.per_token_group_quant_int8(x, 128)
smem_q, smem_s = int8_utils.per_token_group_quant_int8(_misaligned_clone(x), 128)
assert torch.equal(reg_q, smem_q)
assert torch.equal(reg_s, smem_s)
@@ -1,138 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Schema/aliasing tests for the AITER FP8 quantization custom ops.
#
# These use the shared opcheck helper, whose test_schema check catches custom
# ops whose implementation aliases an input that the registered schema declares
# as non-aliasing -- the failure mode behind the rocm_aiter_per_tensor_quant
# regression (a returned scale that aliased the input scale).
#
# Skipped if AITER is not installed or the platform is not ROCm.
import importlib.util
import pytest
import torch
from tests.kernels.utils import opcheck
# this import statement is needed to ensure the ops are registered
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import current_platform
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = pytest.mark.skipif(
not (current_platform.is_rocm() and aiter_available),
reason="AITER ops are only available on ROCm with aiter package installed",
)
FP8_DTYPE = current_platform.fp8_dtype()
def _x(M=128, N=4096):
return torch.randn((M, N), dtype=torch.float16, device="cuda")
def test_per_tensor_quant_static_schema():
"""Static per-tensor: caller provides scale (the aliasing regression)."""
x = _x()
out = torch.empty_like(x, dtype=FP8_DTYPE)
scale = torch.ones(1, dtype=torch.float32, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_per_tensor_quant,
(out, x, scale, False),
)
def test_per_tensor_quant_dynamic_schema():
"""Dynamic per-tensor: op computes scale into the caller's buffer."""
x = _x()
out = torch.empty_like(x, dtype=FP8_DTYPE)
scale = torch.empty(1, dtype=torch.float32, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_per_tensor_quant,
(out, x, scale, True),
)
def test_per_token_quant_dynamic_schema():
"""Dynamic per-token: op computes scale into a freshly allocated buffer."""
x = _x()
opcheck(
torch.ops.vllm.rocm_aiter_per_token_quant,
(x, FP8_DTYPE, None),
)
def test_group_fp8_quant_schema():
"""Dynamic per-token-group quant."""
x = _x()
opcheck(
torch.ops.vllm.rocm_aiter_group_fp8_quant,
(x, 128),
)
@pytest.mark.parametrize("dynamic", [True, False])
def test_per_tensor_quant_matches_native(dynamic):
"""Wrapper output matches the native scaled_fp8_quant reference."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
x = _x()
if dynamic:
scale_in = None
else:
scale_in = torch.tensor([0.5], dtype=torch.float32, device="cuda")
out, scale = rocm_aiter_ops.per_tensor_quant(x, FP8_DTYPE, scale_in)
ref_out, ref_scale = ops.scaled_fp8_quant(x, scale_in)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape == ref_scale.shape
deq = out.to(torch.float32) * scale
if dynamic:
# Dynamic mode: AITER and native each compute their own scale, so their
# outputs differ and can't be compared. Just check that AITER's output
# dequantizes back to the input, within fp8 rounding error.
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
else:
# Static mode: both use the caller's scale, so the outputs must match.
assert torch.equal(scale, scale_in)
ref_deq = ref_out.to(torch.float32) * ref_scale
torch.testing.assert_close(deq, ref_deq, rtol=2e-2, atol=2e-2)
def test_per_token_quant_matches_native():
"""Per-token quant output dequantizes back to the input within FP8 error."""
torch.manual_seed(0)
x = _x()
out, scale = rocm_aiter_ops.per_token_quant(x, FP8_DTYPE)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape[0] == x.shape[0]
# Dequantize and compare to original
deq = out.to(torch.float32) * scale.view(-1, 1)
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
def test_rms_norm_determinism():
"""AITER RMSNorm produces bitwise-identical results across repeated calls."""
# Import to ensure ops are registered
import vllm.kernels.aiter_ops # noqa: F401
torch.manual_seed(0)
M, N = 32, 512
x = torch.randn(M, N, dtype=torch.bfloat16, device="cuda")
weight = torch.ones(N, dtype=torch.bfloat16, device="cuda")
eps = 1e-5
reference = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
for i in range(3):
result = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
assert torch.equal(reference, result), f"Run {i + 1} differs from reference"
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Triton dequant-gather kernel used by
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup)."""
import pytest
import torch
from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_embedding import ( # noqa: E501
_dequant_gather_triton,
)
from vllm.platforms import current_platform
def _dequant_gather_torch(
ids: torch.Tensor,
weight_packed: torch.Tensor,
weight_scale: torch.Tensor,
hidden: int,
num_bits: int,
) -> torch.Tensor:
"""Reference: gather packed rows by id, unpack int32-packed INT, dequant."""
n = ids.shape[0]
int8 = unpack_from_int32(weight_packed[ids], num_bits, torch.Size([n, hidden]))
scale_rows = weight_scale[ids]
w = int8.to(scale_rows.dtype)
if scale_rows.shape[1] == 1:
return w * scale_rows
ng = scale_rows.shape[1]
return (w.view(n, ng, hidden // ng) * scale_rows.unsqueeze(-1)).view(n, hidden)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Triton dequant kernel requires CUDA"
)
@pytest.mark.parametrize("num_bits", [2, 4, 8])
@pytest.mark.parametrize("group_size", [0, 256]) # 0 -> channel
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("num_ids", [1, 17, 4096])
def test_dequant_gather(num_bits, group_size, dtype, num_ids):
torch.manual_seed(0)
device = "cuda"
vocab, hidden = 1000, 2048
pack_factor = 32 // num_bits
# Random full-range int32 packed weights (covers the sign bit -> exercises the
# arithmetic-shift + mask unpack path).
weight_packed = torch.randint(
-(2**31),
2**31,
(vocab, hidden // pack_factor),
dtype=torch.int32,
device=device,
)
num_groups = 1 if group_size == 0 else hidden // group_size
weight_scale = torch.rand(vocab, num_groups, dtype=dtype, device=device) + 0.01
ids = torch.randint(0, vocab, (num_ids,), dtype=torch.long, device=device)
out = _dequant_gather_triton(ids, weight_packed, weight_scale, hidden, num_bits)
ref = _dequant_gather_torch(ids, weight_packed, weight_scale, hidden, num_bits)
assert out.shape == (num_ids, hidden)
assert out.dtype == dtype
torch.testing.assert_close(out, ref)
@@ -1,509 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compile-guard tests for the ROCm RDNA3 W4A16 kernels (dense + MoE).
Verifies that the gfx1100 compilation and dispatch guards are hermetic:
- On gfx1100: all ops exist, dispatch selects RDNA3 kernels.
- On CDNA (gfx942/gfx950) or other non-gfx1100: ops must NOT exist,
dispatch must fall through to Triton/Marlin, and no RDNA3 code
path is reachable.
The negative (non-gfx1100) tests verify at three layers:
1. Compile-level: on non-gfx1100 hardware, the RDNA3 ops are absent
from the compiled _rocm_C extension — real binary verification.
2. Static source analysis: parses CMakeLists.txt and torch_bindings.cpp
to verify that all RDNA3 .cu files and op registrations are inside
gfx1100-only guards.
3. Runtime mock: patches on_gfx1100() to False and verifies that the
Python dispatch chain rejects the RDNA3 path.
Run `pytest tests/kernels/quantization/test_rdna3_compile_guards.py`.
"""
from pathlib import Path
from unittest.mock import patch
import pytest
import regex as re
import torch
import vllm
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 compile-guard tests are ROCm-only", allow_module_level=True)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
kInt4Static,
kInt4Static32,
kInt4Static32Asym,
kInt4StaticAsym,
kInt8Static,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
gfx1100_only = pytest.mark.skipif(
not on_gfx1100(),
reason="Requires gfx1100 hardware",
)
not_gfx1100 = pytest.mark.skipif(
on_gfx1100(),
reason="This test verifies non-gfx1100 builds — skip on gfx1100",
)
RDNA3_OPS = ["gptq_gemm_rdna3", "gptq_gemm_rdna3_wmma", "moe_gptq_gemm_rdna3"]
RDNA3_CU_FILES = [
"q_gemm_rdna3.cu",
"q_gemm_rdna3_wmma.cu",
"moe_q_gemm_rdna3.cu",
]
def _find_repo_root() -> Path | None:
"""Walk up from this file to find the repo root (has CMakeLists.txt)."""
for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
if (parent / "CMakeLists.txt").exists() and (parent / "csrc").is_dir():
return parent
return None
REPO_ROOT = _find_repo_root()
# Directory of the *installed* vllm python package. The .py guard checks read
# from here so they verify the code that is actually imported at runtime — this
# works even on CI images that ship the wheel instead of the python source tree
# (where only csrc/ + CMakeLists.txt are checked out for building).
VLLM_PKG_DIR: Path | None = (
Path(vllm.__file__).parent if getattr(vllm, "__file__", None) else None
)
needs_source = pytest.mark.skipif(
REPO_ROOT is None,
reason="C/CMake source tree not available (installed package only)",
)
def _read_source_or_skip(*relparts: str) -> str:
"""Read a C/CMake source file from the repo tree, or skip if absent.
Used for csrc/ and CMakeLists.txt — these only exist in a source checkout,
not in the installed wheel.
"""
assert REPO_ROOT is not None # callers are gated by @needs_source
path = REPO_ROOT.joinpath(*relparts)
if not path.exists():
pytest.skip(f"{path} not present in this source tree")
return path.read_text()
def _read_pkg_source_or_skip(*relparts: str) -> str:
"""Read a python source file from the installed vllm package.
Reflects the code actually loaded at runtime, so these guard checks run in
CI against the wheel — no source checkout required. Only skips for an
exotic install layout (namespace/zipimport) where __file__ is unavailable.
"""
if VLLM_PKG_DIR is None:
pytest.skip("vllm package directory not resolvable (zip/namespace?)")
assert VLLM_PKG_DIR is not None # narrow for mypy (skip above is NoReturn)
path = VLLM_PKG_DIR.joinpath(*relparts)
if not path.exists():
pytest.skip(f"{path} not present in installed vllm package")
return path.read_text()
# ============================================================================
# Part A: POSITIVE — on gfx1100, ops exist and dispatch works
# ============================================================================
@gfx1100_only
@pytest.mark.parametrize("op_name", RDNA3_OPS)
def test_op_registered_on_gfx1100(op_name):
"""On gfx1100, all RDNA3 ops must be registered in _rocm_C."""
assert hasattr(torch.ops, "_rocm_C"), "_rocm_C module not loaded"
assert hasattr(torch.ops._rocm_C, op_name), (
f"_rocm_C.{op_name} not registered — "
"check CMakeLists.txt VLLM_ROCM_HAS_GFX1100 "
"and torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100"
)
@gfx1100_only
def test_all_ops_present_or_all_absent():
"""The 3 RDNA3 ops are behind the same #ifdef — all present or all absent.
Catches someone accidentally moving an op outside the guard.
"""
has_rocm_c = hasattr(torch.ops, "_rocm_C")
if not has_rocm_c:
pytest.skip("_rocm_C not loaded")
present = {op: hasattr(torch.ops._rocm_C, op) for op in RDNA3_OPS}
values = set(present.values())
assert len(values) == 1, (
f"Guard inconsistency — some RDNA3 ops registered, others not: "
f"{present}. Check torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100 block."
)
# ============================================================================
# Part B: NEGATIVE — compile-level verification on non-gfx1100
# ============================================================================
@not_gfx1100
@pytest.mark.parametrize("op_name", RDNA3_OPS)
def test_op_absent_on_non_gfx1100(op_name):
"""On non-gfx1100 (CDNA), RDNA3 ops must NOT exist in _rocm_C.
This is the real compile-level check: the binary was built without
gfx1100 support, so the ops should not have been compiled or registered.
"""
if not hasattr(torch.ops, "_rocm_C"):
return
assert not hasattr(torch.ops._rocm_C, op_name), (
f"_rocm_C.{op_name} is registered on non-gfx1100 hardware — "
"compile guard is broken: check CMakeLists.txt "
"VLLM_ROCM_HAS_GFX1100 and torch_bindings.cpp #ifdef"
)
@not_gfx1100
def test_rocm_moe_not_supported_on_non_gfx1100():
"""The RDNA3 MoE experts must not be selectable on non-gfx1100 hardware."""
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
Rdna3WNA16Experts,
)
assert Rdna3WNA16Experts._supports_current_device() is False, (
"Rdna3WNA16Experts reported support on non-gfx1100 — dispatch guard is broken"
)
@not_gfx1100
def test_dense_kernel_rejects_on_non_gfx1100():
"""RDNA3W4A16LinearKernel.can_implement must reject on non-gfx1100."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
RDNA3W4A16LinearKernel,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
# ============================================================================
# Part C: Static source analysis (build-level guards)
# ============================================================================
@needs_source
class TestCMakeGuards:
"""Verify CMakeLists.txt only compiles RDNA3 .cu files for gfx1100."""
@staticmethod
def _read_cmake():
return _read_source_or_skip("CMakeLists.txt")
def test_rdna3_cu_files_inside_gfx1100_conditional(self):
"""All RDNA3 .cu files must be listed inside the
``if(VLLM_GPU_ARCHES MATCHES "gfx1100")`` block, not unconditionally.
"""
cmake = self._read_cmake()
for cu_file in RDNA3_CU_FILES:
assert cu_file in cmake, f"{cu_file} not found in CMakeLists.txt"
lines = cmake.splitlines()
in_gfx1100_block = False
for line in lines:
if 'VLLM_GPU_ARCHES MATCHES "gfx1100"' in line:
in_gfx1100_block = True
if in_gfx1100_block and "endif()" in line:
in_gfx1100_block = False
if cu_file in line:
assert in_gfx1100_block, (
f"{cu_file} is listed OUTSIDE the gfx1100 "
f"conditional in CMakeLists.txt — CDNA builds "
f"would compile RDNA3 code. Line: {line.strip()}"
)
def test_compile_definition_only_for_gfx1100(self):
"""VLLM_ROCM_GFX1100 compile definition must be conditional."""
cmake = self._read_cmake()
lines = cmake.splitlines()
in_gfx1100_block = False
for line in lines:
if "VLLM_ROCM_HAS_GFX1100)" in line:
in_gfx1100_block = True
if in_gfx1100_block and "endif()" in line:
in_gfx1100_block = False
if "VLLM_ROCM_GFX1100" in line and "target_compile_definitions" in line:
assert in_gfx1100_block, (
"VLLM_ROCM_GFX1100 compile definition is set outside "
"the VLLM_ROCM_HAS_GFX1100 conditional — CDNA builds "
f"would define it. Line: {line.strip()}"
)
@needs_source
class TestTorchBindingsGuards:
"""Verify torch_bindings.cpp gates all RDNA3 ops behind #ifdef."""
@staticmethod
def _read_bindings():
return _read_source_or_skip("csrc", "rocm", "torch_bindings.cpp")
def test_all_rdna3_ops_inside_ifdef(self):
"""Every rdna3 op def/impl must be between #ifdef VLLM_ROCM_GFX1100
and #endif. If any is outside, a CDNA build would try to register
the op and link a symbol that doesn't exist.
"""
src = self._read_bindings()
lines = src.splitlines()
inside_guard = False
rdna3_lines_outside = []
for i, line in enumerate(lines, 1):
if "#ifdef VLLM_ROCM_GFX1100" in line:
inside_guard = True
elif line.strip() == "#endif" and inside_guard:
inside_guard = False
if (
"rdna3" in line.lower()
and not line.strip().startswith("//")
and not inside_guard
):
rdna3_lines_outside.append((i, line.strip()))
assert not rdna3_lines_outside, (
"RDNA3 op references found OUTSIDE #ifdef VLLM_ROCM_GFX1100 "
"in torch_bindings.cpp — these would break CDNA builds:\n"
+ "\n".join(f" L{n}: {s}" for n, s in rdna3_lines_outside)
)
def test_no_unconditional_rdna3_includes(self):
"""No #include of RDNA3-specific headers outside the guard."""
src = self._read_bindings()
lines = src.splitlines()
inside_guard = False
for i, line in enumerate(lines, 1):
if "#ifdef VLLM_ROCM_GFX1100" in line:
inside_guard = True
elif line.strip() == "#endif" and inside_guard:
inside_guard = False
if "#include" in line and "rdna3" in line.lower():
assert inside_guard, (
f"L{i}: RDNA3 include outside gfx1100 guard: {line.strip()}"
)
class TestCustomOpsGuards:
"""Verify _custom_ops.py gates register_fake behind hasattr checks."""
@staticmethod
def _read_custom_ops():
return _read_pkg_source_or_skip("_custom_ops.py")
def test_register_fake_guarded_by_hasattr(self):
"""Every register_fake for an RDNA3 op must be preceded by a hasattr
check — otherwise it would crash on import on CDNA where the ops
don't exist.
"""
src = self._read_custom_ops()
for op in RDNA3_OPS:
pattern = rf'register_fake\(\s*"_rocm_C::{op}"\s*\)'
match = re.search(pattern, src)
if match is None:
continue
preceding = src[: match.start()]
last_hasattr = preceding.rfind(f'hasattr(torch.ops._rocm_C, "{op}")')
assert last_hasattr != -1, (
f'register_fake("_rocm_C::{op}") is not preceded by a '
f"hasattr check — would crash on CDNA import"
)
gap = preceding[last_hasattr:].count("\n")
assert gap <= 5, (
f"hasattr guard for {op} is {gap} lines before "
f"register_fake — suspiciously far; verify it's the "
f"actual guard and not a coincidence"
)
def test_no_toplevel_rocm_c_import(self):
"""No top-level ``from vllm._rocm_C import`` — would crash on CDNA."""
src = self._read_custom_ops()
for line in src.splitlines():
stripped = line.strip()
if stripped.startswith("#") or stripped.startswith("//"):
continue
assert "from vllm._rocm_C import" not in stripped, (
f"Top-level import of _rocm_C in _custom_ops.py would "
f"crash on CDNA: {stripped}"
)
# ============================================================================
# Part D: Runtime mock (simulate CDNA on gfx1100 hardware)
# ============================================================================
class TestMoEDispatchMocked:
"""Mock on_gfx1100() to False and verify RDNA3 MoE is unreachable."""
def test_kernel_unavailable_when_mocked_cdna(self):
"""The device gate must reject when not on gfx1100."""
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
rdna3_moe_kernel_available,
)
with patch("vllm.platforms.rocm.on_gfx1100", return_value=False):
assert rdna3_moe_kernel_available() is False
@pytest.mark.parametrize(
"weight_key",
[kInt8Static, kInt4StaticAsym, kInt4Static32Asym, None],
)
def test_quant_scheme_rejects_non_symmetric_int4(self, weight_key):
"""Only symmetric int4 weight-only schemes reach the kernel."""
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
Rdna3WNA16Experts,
)
assert Rdna3WNA16Experts._supports_quant_scheme(weight_key, None) is False
@pytest.mark.parametrize("weight_key", [kInt4Static, kInt4Static32])
def test_quant_scheme_accepts_symmetric_int4(self, weight_key):
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
Rdna3WNA16Experts,
)
assert Rdna3WNA16Experts._supports_quant_scheme(weight_key, None) is True
def test_kernel_unavailable_when_op_missing(self):
"""The device gate returns False when the C++ op doesn't exist."""
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
rdna3_moe_kernel_available,
)
fake_rocm_c = type("FakeRocmC", (), {"gptq_gemm_rdna3": None})()
with patch.object(torch, "ops", create=True) as mock_ops:
mock_ops._rocm_C = fake_rocm_c
assert rdna3_moe_kernel_available() is False
def test_kernel_unavailable_when_rocm_c_absent(self):
"""The device gate returns False when _rocm_C doesn't exist at all."""
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
rdna3_moe_kernel_available,
)
fake_ops = type("FakeOps", (), {})()
with patch.object(torch, "ops", fake_ops):
assert rdna3_moe_kernel_available() is False
class TestDenseKernelSelectionMocked:
"""Mock on_gfx1100() and verify dense RDNA3 kernel is not selected."""
@gfx1100_only
def test_can_implement_rejects_when_mocked_cdna(self):
"""RDNA3W4A16LinearKernel.can_implement must reject on mocked CDNA."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
RDNA3W4A16LinearKernel,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
)
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is True
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
patch("vllm.platforms.rocm._ON_GFX1100", False),
):
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is False, f"RDNA3 kernel accepted on simulated CDNA: {reason}"
@gfx1100_only
def test_chooser_skips_rdna3_when_mocked_cdna(self):
"""choose_mp_linear_kernel must NOT return RDNA3 on mocked CDNA."""
from vllm.model_executor.kernels.linear import (
choose_mp_linear_kernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=128,
zero_points=False,
)
with (
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
patch("vllm.platforms.rocm._ON_GFX1100", False),
):
chosen = choose_mp_linear_kernel(config)
assert chosen.__name__ != "RDNA3W4A16LinearKernel", (
"RDNA3 kernel was selected on simulated CDNA — "
"choose_mp_linear_kernel guard is broken"
)
class TestWNA16OracleWiring:
"""The RDNA3 backend must reach the kernel only through the oracle."""
def test_backend_maps_to_rdna3_experts(self):
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
Rdna3WNA16Experts,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
backend_to_kernel_cls,
map_wna16_backend,
)
assert backend_to_kernel_cls(WNA16MoEBackend.RDNA3) == [Rdna3WNA16Experts]
assert map_wna16_backend("rdna3") == WNA16MoEBackend.RDNA3
def test_backend_is_offered_before_the_triton_fallback(self):
"""Priority order: the native kernel outranks Triton when supported."""
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
WNA16MoEBackend,
_get_priority_backends,
)
backends = _get_priority_backends()
assert backends.index(WNA16MoEBackend.RDNA3) < backends.index(
WNA16MoEBackend.TRITON
)
@@ -1,442 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the ROCm RDNA3 fused MoE W4A16 HIP kernel (gfx1100).
Tests ``moe_gptq_gemm_rdna3`` against the dense ``gptq_gemm_rdna3`` as
reference: builds RDNA3-format weights (shuffled int32, synthesized qzeros),
runs the fused MoE kernel, and compares per-expert results.
Model parameters taken from:
- cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit
(hidden=2048, inter=768, E=128, top_k=8, G=32)
- Qwen3.6-35B-A3B-GPTQ-W4A16-G32
(hidden=2048, inter=512, E=256, top_k=8, G=32)
Run `pytest tests/kernels/quantization/test_rdna3_moe_w4a16.py`.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 MoE W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm import _custom_ops as ops # noqa: E402
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
MoEActivation,
apply_moe_activation,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( # noqa: E402
moe_align_block_size,
)
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( # noqa: E402
_process_weights_rdna3,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
pack_quantized_values_into_int32,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
device = "cuda"
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "moe_gptq_gemm_rdna3")
),
reason="Requires gfx1100 with moe_gptq_gemm_rdna3 op",
)
# Model configurations: real K/N/top_k/group_size dims, E capped at 16 to
# fit in test GPU memory (full E=128/256 would need >20GB for weights alone).
# Kernel behavior is E-independent (per-expert tiling), so E=16 is sufficient.
MODEL_CONFIGS = [
# cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit dims (E capped)
pytest.param(16, 2048, 768, 8, 32, id="Qwen3-30B-A3B"),
# Qwen3.6-35B-A3B-GPTQ-W4A16-G32 dims (E capped)
pytest.param(16, 2048, 512, 8, 32, id="Qwen3.6-35B-A3B"),
]
# Token counts: decode (1), small batch (4), medium (16), prefill (64)
NUM_TOKENS = [1, 4, 16, 64, 256, 512]
def _make_packed_weights(E, K, N):
"""Create random 4-bit packed weights [E, K/8, N] int32 + shuffle."""
w = torch.randint(0, 16, (E, K, N), dtype=torch.int32, device=device)
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
for i in range(8):
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
for e in range(E):
we = packed[e].contiguous()
ops.gptq_shuffle(we, 4)
packed[e] = we
return packed
def _make_scales(E, groups, N, dtype):
return torch.rand(E, groups, N, dtype=dtype, device=device) * 0.1
def _make_qzeros(E, groups, N):
zeros = torch.full(
(groups, N),
scalar_types.uint4b8.bias - 1,
dtype=torch.int32,
device=device,
)
qz = pack_quantized_values_into_int32(
zeros,
scalar_types.uint4b8,
packed_dim=1,
)
return qz.unsqueeze(0).expand(E, -1, -1).contiguous()
def _make_ct_weights(E, N, K, group_size, dtype):
"""Compressed-tensors canonical N-first layout: packed ``[E, N, K // 8]``
and scales ``[E, N, K // group_size]``, plus the dequantized weights."""
q = torch.randint(0, 16, (E, N, K), dtype=torch.int32, device=device)
s = torch.rand(E, N, K // group_size, dtype=dtype, device=device) * 0.1 + 0.01
packed = pack_quantized_values_into_int32(q, scalar_types.uint4b8, packed_dim=2)
w = (q - 8).float() * s.float().repeat_interleave(group_size, dim=2)
return packed.contiguous(), s, w
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("block_size_m", [1, 4])
def test_fused_moe_w1_matches_dense(
E, K, N_inter, top_k, group_size, M, dtype, block_size_m
):
"""w1 GEMM via fused kernel matches per-expert dense kernel."""
N_gate_up = N_inter * 2
groups = K // group_size
torch.manual_seed(42)
x = torch.randn(M, K, dtype=dtype, device=device)
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, groups, N_gate_up, dtype)
w13_z = _make_qzeros(E, groups, N_gate_up)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
# Fused kernel
fused_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
fused_out,
w13,
w13_s,
w13_z,
torch.empty(0, device=device),
si,
ei,
ntp,
top_k,
block_size_m,
False,
0,
)
# Per-expert dense reference
ref_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
for m in range(M):
for k in range(top_k):
e = topk_ids[m, k].item()
flat = m * top_k + k
ref = ops.gptq_gemm_rdna3(
x[m : m + 1],
w13[e],
w13_z[e],
w13_s[e],
False,
)
ref_out[flat] = ref.squeeze()
# Split-K atomics can cause minor fp16/bf16 rounding differences
# at large K (e.g. K=2048 → 8 K-blocks). Use allclose, not equal.
atol = 0.5 if dtype == torch.bfloat16 else 0.1
assert torch.allclose(fused_out, ref_out, atol=atol, rtol=0.01), (
f"max diff: {(fused_out - ref_out).abs().max().item()}"
)
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_fused_moe_output_topk_reduces(E, K, N_inter, top_k, group_size, M, dtype):
"""output_topk fuses moe_sum: multiple experts write to same output row."""
groups = K // group_size
torch.manual_seed(123)
x = torch.randn(M * top_k, K, dtype=dtype, device=device)
w = _make_packed_weights(E, K, N_inter)
ws = _make_scales(E, groups, N_inter, dtype)
wz = _make_qzeros(E, groups, N_inter)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
torch.randn(M, top_k, device=device),
dim=-1,
).float()
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
# Without output_topk: write to [M*top_k, N] then moe_sum
flat_out = torch.zeros(M * top_k, N_inter, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
flat_out,
w,
ws,
wz,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
0,
)
ref = torch.zeros(M, N_inter, dtype=dtype, device=device)
ops.moe_sum(flat_out.view(M, top_k, N_inter), ref)
# With output_topk: write directly to [M, N]
fused = torch.zeros(M, N_inter, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
fused,
w,
ws,
wz,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
top_k,
)
atol = 1.0 if dtype == torch.bfloat16 else 0.1
assert torch.allclose(fused, ref, atol=atol, rtol=0.01), (
f"max diff: {(fused - ref).abs().max().item()}"
)
@gfx1100_only
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
@pytest.mark.parametrize("M", NUM_TOKENS)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
"""Full MoE forward: w1 + silu_and_mul + w2 with output_topk reduce."""
N_gate_up = N_inter * 2
hidden = K
torch.manual_seed(7)
x = torch.randn(M, K, dtype=dtype, device=device)
w13 = _make_packed_weights(E, K, N_gate_up)
w13_s = _make_scales(E, K // group_size, N_gate_up, dtype)
w13_z = _make_qzeros(E, K // group_size, N_gate_up)
w2 = _make_packed_weights(E, N_inter, hidden)
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(
torch.randn(M, top_k, device=device),
dim=-1,
).float()
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
# Fused path (what apply() does)
w1_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
w1_out,
w13,
w13_s,
w13_z,
torch.empty(0, device=device),
si,
ei,
ntp,
top_k,
1,
False,
0,
)
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
fused = torch.zeros(M, hidden, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
act_out,
fused,
w2,
w2_s,
w2_z,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
top_k,
)
# Per-expert reference
ref = torch.zeros(M, hidden, dtype=dtype, device=device)
for m_idx in range(M):
for k_idx in range(top_k):
e = topk_ids[m_idx, k_idx].item()
w = topk_w[m_idx, k_idx].item()
r1 = ops.gptq_gemm_rdna3(
x[m_idx : m_idx + 1],
w13[e],
w13_z[e],
w13_s[e],
False,
)
a = torch.empty(1, N_inter, dtype=dtype, device=device)
apply_moe_activation(MoEActivation.SILU, a, r1)
r2 = ops.gptq_gemm_rdna3(
a,
w2[e],
w2_z[e],
w2_s[e],
False,
)
ref[m_idx] += r2.squeeze() * w
# E2E chains w1 + activation + w2 + topk_w + output_topk reduce.
# Each step accumulates rounding error (split-K atomics, topk_w
# multiply order). Use relative L2 norm like the dense kernel test.
diff_l2 = torch.norm(fused.float() - ref.float())
ref_l2 = torch.norm(ref.float())
rel_l2 = (diff_l2 / ref_l2).item() if ref_l2 > 0 else 0.0
threshold = 0.05 if dtype == torch.float16 else 0.10
assert rel_l2 < threshold, (
f"rel L2 = {rel_l2:.4f} (threshold {threshold}), "
f"max abs diff: {(fused - ref).abs().max().item()}"
)
@gfx1100_only
def test_expert_id_minus_one():
"""Kernel handles expert_id == -1 (expert parallelism) without crash."""
# Qwen3-30B-A3B dims (E capped for memory)
E, K, N = 16, 2048, 768
groups = K // 32
w = _make_packed_weights(E, K, N)
ws = _make_scales(E, groups, N, torch.bfloat16)
wz = _make_qzeros(E, groups, N)
x = torch.randn(1, K, dtype=torch.bfloat16, device=device)
# Manually create sorted_token_ids/expert_ids with -1
sorted_ids = torch.tensor([0], dtype=torch.int32, device=device)
expert_ids = torch.tensor([-1], dtype=torch.int32, device=device)
ntp = torch.tensor([1], dtype=torch.int32, device=device)
out = torch.zeros(1, N, dtype=torch.bfloat16, device=device)
ops.moe_gptq_gemm_rdna3(
x,
out,
w,
ws,
wz,
torch.empty(0, device=device),
sorted_ids,
expert_ids,
ntp,
1,
1,
False,
0,
)
current_platform.synchronize()
# Output should remain zero (expert skipped)
assert torch.equal(out, torch.zeros_like(out))
@gfx1100_only
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_process_weights_from_compressed_tensors_layout(dtype):
"""Weights in the layout compressed-tensors hands to the oracle go through
the RDNA3 post-processing and a full MoE forward, against a dequantized
reference. The tests above build the kernel layout directly, so they do
not notice when the layout upstream of the kernel changes (#52798)."""
E, K, N_inter, top_k, group_size, M = 8, 512, 256, 2, 32, 16
torch.manual_seed(5)
w13, w13_s, w13_ref = _make_ct_weights(E, 2 * N_inter, K, group_size, dtype)
w2, w2_s, w2_ref = _make_ct_weights(E, K, N_inter, group_size, dtype)
w13, w2, w13_s, w2_s, w13_z, w2_z, *_ = _process_weights_rdna3(
w13, w2, w13_s, w2_s, group_size
)
assert w13_z is not None and w2_z is not None
x = torch.randn(M, K, dtype=dtype, device=device)
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
topk_w = torch.softmax(torch.randn(M, top_k, device=device), dim=-1).float()
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
w1_out = torch.zeros(M * top_k, 2 * N_inter, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
x,
w1_out,
w13,
w13_s,
w13_z,
torch.empty(0, device=device),
si,
ei,
ntp,
top_k,
1,
False,
0,
)
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
out = torch.zeros(M, K, dtype=dtype, device=device)
ops.moe_gptq_gemm_rdna3(
act_out,
out,
w2,
w2_s,
w2_z,
topk_w.view(-1),
si,
ei,
ntp,
1,
1,
True,
top_k,
)
ref = torch.zeros(M, K, device=device)
for m in range(M):
for k in range(top_k):
e = int(topk_ids[m, k])
h = w13_ref[e] @ x[m].float()
a = F.silu(h[:N_inter]) * h[N_inter:]
ref[m] += (w2_ref[e] @ a) * topk_w[m, k]
rel_l2 = (torch.norm(out.float() - ref) / torch.norm(ref)).item()
assert rel_l2 < (0.02 if dtype == torch.float16 else 0.05), rel_l2
@@ -1,276 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the ROCm RDNA3 W4A16 GPTQ kernel (gfx1100).
Exercises ``RDNA3W4A16LinearKernel`` end-to-end: it builds a layer with
GPTQ-format checkpoint parameters, runs ``process_weights_after_loading``
(weight shuffle + zero-point synthesis), then ``apply_weights``, and compares
the result against an fp32 reference dequant-and-matmul.
The kernel is exposed via ``torch.ops._rocm_C.gptq_gemm_rdna3`` and is only
built for gfx11; tests are skipped elsewhere.
Run `pytest tests/kernels/quantization/test_rdna3_w4a16.py`.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
RDNA3W4A16LinearKernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
pack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import ( # noqa: E402
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
from vllm.utils.torch_utils import set_random_seed # noqa: E402
device = "cuda"
WEIGHT_TYPE = scalar_types.uint4b8 # symmetric int4, bias = 8
PACK_FACTOR = 8 # 8 x 4-bit nibbles per int32
# Skip everything in this module unless we are on the only architecture the
# kernel is built/registered for.
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
),
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
)
# ---------------------------------------------------------------------------
# Reference implementation
# ---------------------------------------------------------------------------
def _reference(
x_mk: torch.Tensor,
q_int4_kn: torch.Tensor,
scales_gn: torch.Tensor,
zeros_gn: torch.Tensor | None,
group_size: int,
bias: torch.Tensor | None,
) -> torch.Tensor:
"""fp32 reference for the RDNA3 W4A16 op.
x_mk: [M, K] fp16/bf16 activations.
q_int4_kn: [K, N] int32 raw stored nibbles in [0, 15].
scales_gn: [K//G, N] per-group scales (act dtype).
zeros_gn: [K//G, N] int32 raw stored zero points in [0, 15], or None
for the symmetric path (kernel synthesizes stored zero = 7).
group_size: G.
The kernel applies the GPTQv1 "+1" zero-point quirk, so the effective
zero is ``stored_zero + 1`` (symmetric path: 7 + 1 == bias == 8).
"""
K, N = q_int4_kn.shape
s_full = scales_gn.repeat_interleave(group_size, dim=0).to(torch.float32) # [K,N]
if zeros_gn is None:
z_full = torch.full(
(K, N), float(WEIGHT_TYPE.bias), device=x_mk.device, dtype=torch.float32
)
else:
z_full = (zeros_gn + 1).repeat_interleave(group_size, dim=0).to(torch.float32)
w_fp = (q_int4_kn.to(torch.float32) - z_full) * s_full # [K, N]
out = x_mk.to(torch.float32) @ w_fp # [M, N]
if bias is not None:
out = out + bias.to(torch.float32)
return out.to(x_mk.dtype)
# ---------------------------------------------------------------------------
# Layer construction (GPTQ checkpoint format)
# ---------------------------------------------------------------------------
def _build_layer(
q_int4_kn: torch.Tensor,
scales_gn: torch.Tensor,
zeros_gn: torch.Tensor | None,
dtype: torch.dtype,
) -> torch.nn.Module:
"""Build a dummy layer carrying GPTQ-format params, as the loader would."""
no_loader = lambda *args, **kwargs: None # noqa: E731
# qweight: int4 packed along K into int32 -> [K//8, N].
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"qweight",
PackedvLLMParameter(
data=qweight,
weight_loader=no_loader,
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=PACK_FACTOR,
),
)
layer.register_parameter(
"scales",
GroupQuantScaleParameter(
data=scales_gn.to(dtype),
weight_loader=no_loader,
input_dim=0,
output_dim=1,
),
)
if zeros_gn is not None:
# qzeros: int4 packed along N into int32 -> [K//G, N//8].
qzeros = pack_quantized_values_into_int32(zeros_gn, WEIGHT_TYPE, packed_dim=1)
layer.register_parameter(
"qzeros",
PackedvLLMParameter(
data=qzeros,
weight_loader=no_loader,
input_dim=0,
output_dim=1,
packed_dim=1,
packed_factor=PACK_FACTOR,
),
)
return layer
def _run_kernel(
x_mk: torch.Tensor,
q_int4_kn: torch.Tensor,
scales_gn: torch.Tensor,
zeros_gn: torch.Tensor | None,
group_size: int,
bias: torch.Tensor | None,
dtype: torch.dtype,
) -> torch.Tensor:
K, N = q_int4_kn.shape
has_zp = zeros_gn is not None
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=WEIGHT_TYPE,
act_type=dtype,
group_size=group_size,
zero_points=has_zp,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok, f"can_implement rejected a supported config: {reason}"
layer = _build_layer(q_int4_kn, scales_gn, zeros_gn, dtype)
kernel = RDNA3W4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros" if has_zp else None,
)
kernel.process_weights_after_loading(layer)
return kernel.apply_weights(layer, x_mk, bias=bias)
# Relative-L2 tolerance per dtype. The bf16 path widens dequantized weights
# to fp32 and accumulates in fp32, so it matches the reference almost exactly
# (<0.4% incl. the WMMA prefill path). The fp16 path uses the exllamav2
# "+1024" bit-trick (see qdq_4_rdna3.cuh): the dequantized weight is recovered
# as the fp16 difference of two ~1024*scale magnitudes, which sheds low-order
# mantissa bits and leaves ~2-3% relative noise that accumulates over K. We
# compare on the relative Frobenius norm rather than elementwise, since the
# bit-trick noise produces large *relative* errors on individual near-zero
# outputs that carry negligible absolute weight.
_REL_L2_TOL = {torch.float16: 5e-2, torch.bfloat16: 1e-2}
def _assert_close(out: torch.Tensor, ref: torch.Tensor, dtype: torch.dtype):
rel_l2 = (out.to(torch.float32) - ref.to(torch.float32)).norm() / ref.to(
torch.float32
).norm()
tol = _REL_L2_TOL[dtype]
assert rel_l2 < tol, f"relative L2 error {rel_l2:.4f} exceeds {tol} for {dtype}"
# ---------------------------------------------------------------------------
# Forward correctness
# ---------------------------------------------------------------------------
# (M, K, N, group_size). M spans the scalar decode path (small M) and the
# WMMA prefill path (M >= 16 on the bf16 dispatch). K/N satisfy the kernel's
# divisibility constraints (K % G == 0, K % 8 == 0, N % 8 == 0).
MKNG_SHAPES = [
(1, 128, 128, 128), # single group, decode
(2, 256, 256, 128), # two groups
(8, 256, 512, 64), # M=8 scalar, smaller group
(16, 512, 256, 128), # M=16 -> WMMA path for bf16
(32, 512, 512, 64), # larger prefill
]
@gfx1100_only
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True], ids=["no_zp", "with_zp"])
@pytest.mark.parametrize(
"M,K,N,G", MKNG_SHAPES, ids=[f"m{m}_k{k}_n{n}_g{g}" for m, k, n, g in MKNG_SHAPES]
)
def test_rdna3_w4a16_matches_reference(dtype, has_zp, M, K, N, G, dist_init):
set_random_seed(0)
assert K % G == 0 and K % PACK_FACTOR == 0 and N % PACK_FACTOR == 0
groups = K // G
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
q_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
scales_gn = (
0.05 * torch.rand((groups, N), device=device, dtype=torch.float32) + 0.01
).to(dtype)
zeros_gn = (
torch.randint(0, 16, (groups, N), device=device, dtype=torch.int32)
if has_zp
else None
)
out = _run_kernel(x_mk, q_int4_kn, scales_gn, zeros_gn, G, None, dtype)
ref = _reference(x_mk, q_int4_kn, scales_gn, zeros_gn, G, None)
assert out.shape == (M, N) and out.dtype == dtype
_assert_close(out, ref, dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("M", [1, 32], ids=["decode", "prefill"])
def test_rdna3_w4a16_bias(dtype, M, dist_init):
"""Bias is added on both the scalar (M=1) and WMMA (M=32) paths."""
set_random_seed(0)
K, N, G = 512, 256, 128
groups = K // G
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
q_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
scales_gn = (
0.05 * torch.rand((groups, N), device=device, dtype=torch.float32) + 0.01
).to(dtype)
bias = (0.1 * torch.randn(N, device=device, dtype=torch.float32)).to(dtype)
out = _run_kernel(x_mk, q_int4_kn, scales_gn, None, G, bias, dtype)
ref = _reference(x_mk, q_int4_kn, scales_gn, None, G, bias)
_assert_close(out, ref, dtype)
@@ -1,585 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Determinism + FP32-reference correctness tests for the RDNA3 W4A16 GEMM.
The split-K epilogues of ``gptq_gemm_rdna3`` (scalar path) and the WMMA
path previously accumulated per-split low-precision partials directly into
the output via CAS atomics. With multiple writers per output element the
accumulation order — and therefore the rounded result — varied per call:
repeated fixed-input calls produced a different output almost every time on
gfx1100 once more than a few split blocks contended.
Both paths now store FP32 split partials and reduce them in a fixed order
with a single final low-precision rounding, which is deterministic by
construction. When the split count is 1 every path stores directly
(single writer per output element).
These tests assert, through the public op path:
A. Repeatability — ``torch.equal`` across repeated identical calls.
B. Numerical correctness — against an FP32 dequantized reference.
(B) exists because repeatability alone cannot see the failure mode the
deterministic epilogue introduces: the FP32 partials scratch is
``at::empty`` (its coverage invariant is documented at
``alloc_wmma_partials`` in ``csrc/rocm/q_gemm_rdna3_wmma.cu``), and across
repeated calls the caching allocator hands back the same block — a slot
that ever went unwritten would be bitwise-repeatable *and* wrong.
Reference semantics (uint4b8, GPTQv1, synthesized zero points):
W[k, n] = (q[k, n] - 8) * float(round_to_dtype(scale[g, n])), g = k//G
ref[m, n] = x[m, :] @ W[:, n] (FP32 matmul)
Tolerances are derived from the kernel's rounding structure, not tuned to
pass (measured max errors on W7900/gfx1100 in parentheses):
* scalar bf16 — dequant keeps FP32 precision end-to-end (magic-value
v_dot2 path); the only rounding is the final cast. Measured max ~0.06
on the tested shapes (|ref| up to ~30); tolerance 0.10 with headroom
for larger |ref| at the final-cast ulp.
* scalar fp16 — the dequant uses the exact factored form (magic values in
fp16, (y, z) correction in fp32; see qdq_4_rdna3.cuh), so the only fp16
roundings are the final output cast and the B-tile-free dot inputs:
max_abs <= 0.5 * ulp_fp16(max|ref|) * 1.5 + 1e-3 (measured ~0.015 at
|ref|~37; the pre-fix baked-constant form measured ~1.0-1.4 on the same
shapes — 94-200x worse).
* WMMA bf16 — B is narrowed to bf16 per cell (one rounding,
<= 0.5*ulp_bf16(|B|)), accumulated in FP32; measured max ≈ 0.09 at
K=4096 — tolerance 0.25.
* WMMA fp16 — B narrowed to fp16 via the precise sub-then-mul variant
(one rounding, ~1e-4); measured max ≈ 0.015 — tolerance 0.05.
All four are orders of magnitude below what an unwritten scratch slot
(caching-allocator garbage / NaN) or an indexing bug (errors of the same
scale as |ref| ≈ 7) would produce.
Test hygiene: the op path needs a single-process distributed group and a
current VLLM config; tests use the repo-standard ``dist_init`` fixture from
``tests/conftest.py`` (real context manager, temp-file rendezvous so no
hardcoded MASTER_PORT can collide in parallel CI, and the suite's standard
post-test cleanup).
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
RDNA3W4A16LinearKernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
pack_quantized_values_into_int32,
)
from vllm.model_executor.parameter import ( # noqa: E402
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
device = "cuda"
WEIGHT_TYPE = scalar_types.uint4b8
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
),
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
)
REPEATS = 20
GROUP = 128
DTYPES = [torch.bfloat16, torch.float16]
# max_abs_error tolerances vs the FP32 reference, per (path, dtype).
# Derivation in the module docstring; ~2-3x headroom over measured maxima.
TOL = {
("scalar", torch.bfloat16): 0.10,
("scalar", torch.float16): 0.025,
("wmma", torch.bfloat16): 0.25,
("wmma", torch.float16): 0.05,
}
def _build_layer(k, n, seed, dtype, group=GROUP):
torch.manual_seed(seed)
q_int4_kn = torch.randint(0, 16, (k, n), dtype=torch.int32)
scales_gn = (torch.randn(k // group, n) * 0.01 + 0.02).to(dtype)
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
no_loader = lambda *a, **kw: None # noqa: E731
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"qweight",
PackedvLLMParameter(
data=qweight,
weight_loader=no_loader,
input_dim=0,
output_dim=1,
packed_dim=0,
packed_factor=8,
),
)
layer.register_parameter(
"scales",
GroupQuantScaleParameter(
data=scales_gn, weight_loader=no_loader, input_dim=0, output_dim=1
),
)
layer.to(device)
return layer, q_int4_kn, scales_gn
def _prepare(layer, dtype, k, n, group=GROUP):
cfg = MPLinearLayerConfig(
full_weight_shape=(k, n),
partition_weight_shape=(k, n),
weight_type=WEIGHT_TYPE,
act_type=dtype,
group_size=group,
zero_points=False,
)
kernel = RDNA3W4A16LinearKernel(
cfg, w_q_param_name="qweight", w_s_param_name="scales", w_zp_param_name=None
)
kernel.process_weights_after_loading(layer)
return kernel._get_weight_params(layer)
def _reference(m, k, dtype, seed, q_int4_kn, scales_gn, group=GROUP):
torch.manual_seed(seed + 1)
x = torch.randn(m, k, device=device, dtype=dtype)
# FP32 reference; see module docstring for the dequant semantics.
w_f32 = (q_int4_kn.to(device).float() - 8.0) * scales_gn.to(
device
).repeat_interleave(group, dim=0).float()
return x, x.float() @ w_f32
def _run_op(x, w_q, w_zp, w_s):
return torch.ops._rocm_C.gptq_gemm_rdna3(x, w_q, w_zp, w_s, False)
def _outputs_and_ref(M, K, N, seed, dtype, repeats=1, group=GROUP):
layer, q_int4_kn, scales_gn = _build_layer(K, N, seed, dtype, group)
w_q, w_s, w_zp = _prepare(layer, dtype, K, N, group)
x, ref = _reference(M, K, dtype, seed, q_int4_kn, scales_gn, group)
outs = [_run_op(x, w_q, w_zp, w_s) for _ in range(repeats)]
return outs, ref
def _path_for(dtype, m):
"""Public dispatch: both dtypes reach WMMA at M>=12 (see the
WMMA_MIN_M crossover table in csrc/rocm/q_gemm_rdna3.cu)."""
if m >= 12:
return "wmma"
return "scalar"
def _assert_repeatable(outs):
for o in outs[1:]:
assert torch.equal(o, outs[0]), (
"RDNA3 W4A16 split-K produced differing outputs for identical inputs"
)
def _assert_close_to_ref(out, ref, path, dtype):
err = (out.float() - ref).abs()
max_abs = err.max().item()
mean_abs = err.mean().item()
tol = TOL[(path, dtype)]
assert torch.isfinite(out.float()).all(), "non-finite output"
assert max_abs <= tol, (
f"RDNA3 W4A16 {path} output deviates from the FP32 dequantized "
f"reference: max_abs={max_abs:.4f} > tol={tol} (mean_abs="
f"{mean_abs:.4f}). This catches values that are bitwise-repeatable "
"but wrong (e.g. an unwritten FP32 scratch slot)."
)
# ---------------------------------------------------------------------------
# A. Repeatability (bit-exact across identical calls)
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_splitk_bit_repeatable(dist_init, dtype):
"""Scalar path at production-like contention: K=4096 splits into
ceil(4096/256) = 16 concurrent writers per output element (the old CAS
bug's onset was between 2 and 4 writers, so this sits well inside the
failure regime rather than at the empirical threshold)."""
outs, _ = _outputs_and_ref(
M=1, K=4096, N=4096, seed=1234, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_small_m_bit_repeatable(dist_init, dtype):
"""Scalar M>1 tiles share the same deterministic epilogue."""
outs, _ = _outputs_and_ref(
M=8, K=4096, N=4096, seed=1235, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_single_split_bit_repeatable(dist_init, dtype):
"""z_count == 1 (K=256 < BLOCK_KN_SIZE): direct-store epilogue, no
scratch, no reduce pass."""
outs, _ = _outputs_and_ref(
M=1, K=256, N=4096, seed=1238, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_splitk_bit_repeatable(dist_init, dtype):
"""WMMA path (both dtypes M >= 12), split-K active.
K=6656 gives K_SPLIT=4 under the upstream heuristic; the deterministic
epilogue must be bit-repeatable regardless of the split count.
"""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, _ = _outputs_and_ref(
M=m, K=6656, N=4096, seed=1236, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_large_m_bit_repeatable(dist_init, dtype):
"""Large-M WMMA tiles (128x64 kernels) with the deterministic epilogue."""
outs, _ = _outputs_and_ref(
M=128, K=6656, N=4096, seed=1237, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
# ---------------------------------------------------------------------------
# B. Numerical correctness against the FP32 dequantized reference
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_matches_fp32_reference(dist_init, dtype):
"""Scalar split-K (K=4096, 16 writers) vs the FP32 reference."""
m = 1
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1240, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("m", [2, 3, 4, 5, 8])
def test_scalar_multi_m_matches_fp32_reference(dist_init, m, dtype):
"""The exact factored dequant across every M_COUNT instantiation
(M=2,3 -> M_COUNT=2; M=4,5 -> M_COUNT=4 with a padded row; M=8 -> 8):
the (y, z) correction and the nibble-slot activation sums must hold
for every tile width, not just the M=1 fast path."""
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1247, dtype=dtype)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_single_split_matches_fp32_reference(dist_init, dtype):
"""Scalar z_count == 1 direct-store path vs the FP32 reference."""
m = 1
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1241, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_matches_fp32_reference(dist_init, dtype):
"""WMMA split-K path (16x16_1w at the M=12 boundary) vs FP32 reference."""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1242, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_single_split_matches_fp32_reference(dist_init, dtype):
"""WMMA k_split == 1 direct-store path (K=256 -> compute_wmma_k_split
returns 1 for every dispatch level) vs the FP32 reference."""
m = 12 # smallest M on the WMMA side of the dispatch boundary
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1243, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_large_m_matches_fp32_reference(dist_init, dtype):
"""Large-M WMMA (128x64_k32, row-tiled scratch + reduce) vs FP32."""
m = 128
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1244, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
# ---------------------------------------------------------------------------
# C. V7/V8 (128x64) no-split path: k_split == 1 -> direct store, no scratch,
# no reduce pass (partials == nullptr).
# ---------------------------------------------------------------------------
# The measured production-like shape from the W7900 benchmark campaign.
# Routing arithmetic (see _assert_v7v8_no_split_routing): with (128, 64)
# tiles, blocks_xy = ceil(25600/64) * ceil(512/128) = 400 * 4 = 1600 >=
# 1500, so compute_wmma_k_split_mn returns 1. Smaller no-split shapes are
# not practical: N would have to grow past ~24k at M=512 (or M past ~1k at
# N=12k) to reach the 1500-block threshold, so this is effectively the
# smallest production-realistic V7/V8 no-split shape.
V7V8_NOSPLIT = (512, 25600, 6656) # (M, N, K)
def _compute_wmma_k_split_mn(m, n, k, m_tile, n_tile):
"""Faithful replica of compute_wmma_k_split_mn (C++ heuristic)."""
blocks_xy = ((n + n_tile - 1) // n_tile) * ((m + m_tile - 1) // m_tile)
if blocks_xy >= 1500:
return 1
if blocks_xy * 2 >= 1500 and k >= 512 and k % 32 == 0:
return 2
if blocks_xy * 4 >= 1500 and k >= 1024 and k % 64 == 0:
return 4
if k >= 1024 and k % 64 == 0:
return 4
if k >= 512 and k % 32 == 0:
return 2
return 1
def _assert_v7v8_no_split_routing(dtype, m, n, k):
"""Assert the shape deterministically routes to the V7/V8 128x64
kernel with k_split == 1: WMMA dispatch (both dtypes M >= 12), the
M >= 128 branch, and the no-split threshold."""
assert m >= 12, "not the WMMA path"
assert m >= 128, "below the V7/V8 (128x64) branch"
assert _compute_wmma_k_split_mn(m, n, k, 128, 64) == 1, (
"shape does not route to the k_split == 1 no-split path"
)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_v7v8_no_split_bit_repeatable(dist_init, dtype):
"""V7/V8 128x64_k32 with k_split == 1: single direct writer per output
cell — no FP32 scratch, no reduce pass."""
m, n, k = V7V8_NOSPLIT
_assert_v7v8_no_split_routing(dtype, m, n, k)
outs, _ = _outputs_and_ref(M=m, K=k, N=n, seed=1245, dtype=dtype, repeats=REPEATS)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_wmma_v7v8_no_split_matches_fp32_reference(dist_init, dtype):
"""V7/V8 128x64_k32 no-split direct-store path vs the FP32 reference."""
m, n, k = V7V8_NOSPLIT
_assert_v7v8_no_split_routing(dtype, m, n, k)
outs, ref = _outputs_and_ref(M=m, K=k, N=n, seed=1246, dtype=dtype)
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
# ---------------------------------------------------------------------------
# D. Dispatch boundary: M=11 must stay scalar, M=12 must route to WMMA
# (WMMA_MIN_M in csrc/rocm/q_gemm_rdna3.cu). Observed via the CUDA
# profiler: the scalar and WMMA kernels have disjoint names.
# ---------------------------------------------------------------------------
def _launched_kernel_names(fn) -> set:
from torch.profiler import ProfilerActivity, profile
fn()
torch.accelerator.synchronize()
with profile(activities=[ProfilerActivity.CUDA]) as prof:
fn()
torch.accelerator.synchronize()
return {
e.name
for e in prof.events()
if "gemm_q4" in e.name or "reduce_partials" in e.name
}
@pytest.mark.parametrize("dtype", DTYPES)
@gfx1100_only
def test_dispatch_boundary(dist_init, dtype):
"""The scalar/WMMA crossover sits between M=11 and M=12 for both
dtypes; the boundary must be exact (see the WMMA_MIN_M table)."""
def run(m):
outs, _ = _outputs_and_ref(M=m, K=4096, N=512, seed=1248, dtype=dtype)
return outs[0]
m11 = run(11)
names11 = _launched_kernel_names(lambda: run(11))
assert any("gemm_q4_kernel_rdna3" in n for n in names11), names11
assert not any("wmma" in n for n in names11), names11
m12 = run(12)
names12 = _launched_kernel_names(lambda: run(12))
assert any("wmma" in n for n in names12), names12
assert not any("gemm_q4_kernel_rdna3" in n for n in names12), names12
# Both sides of the boundary stay correct against the reference.
_, ref11 = _outputs_and_ref(M=11, K=4096, N=512, seed=1248, dtype=dtype)
_, ref12 = _outputs_and_ref(M=12, K=4096, N=512, seed=1248, dtype=dtype)
_assert_close_to_ref(m11, ref11, "scalar", dtype)
_assert_close_to_ref(m12, ref12, "wmma", dtype)
# ---------------------------------------------------------------------------
# E. In-kernel split-K reduction: stress the per-call ticket epilogue.
# The fused reduction must stay deterministic and correct under
# back-to-back unsynchronized calls, concurrent invocations on two
# streams (per-call tickets must never share state), and CUDA-graph
# capture/replay.
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_splitk_100x_bit_repeatable(dist_init, dtype):
"""100 back-to-back calls (no intermediate sync) on the split-K path."""
outs, _ = _outputs_and_ref(M=1, K=4096, N=512, seed=1249, dtype=dtype, repeats=100)
_assert_repeatable(outs)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_multistream_concurrent(dist_init, dtype):
"""Two concurrent invocations on separate streams, distinct weights and
activations: outputs must be bit-identical to the serial invocation of
each (the arrival tickets are per-call, so cross-stream interference
would show up here as a corrupted or unreduced output)."""
layer_a, q_a, s_a = _build_layer(4096, 512, 4321, dtype)
layer_b, q_b, s_b = _build_layer(4096, 512, 4322, dtype)
wq_a, ws_a, wz_a = _prepare(layer_a, dtype, 4096, 512)
wq_b, ws_b, wz_b = _prepare(layer_b, dtype, 4096, 512)
torch.manual_seed(4323)
x_a = torch.randn(1, 4096, device=device, dtype=dtype)
x_b = torch.randn(1, 4096, device=device, dtype=dtype)
serial_a = _run_op(x_a, wq_a, wz_a, ws_a)
serial_b = _run_op(x_b, wq_b, wz_b, ws_b)
torch.accelerator.synchronize()
s1, s2 = torch.cuda.Stream(), torch.cuda.Stream()
for _ in range(25):
with torch.cuda.stream(s1):
out_a = _run_op(x_a, wq_a, wz_a, ws_a)
with torch.cuda.stream(s2):
out_b = _run_op(x_b, wq_b, wz_b, ws_b)
torch.accelerator.synchronize()
assert torch.equal(out_a, serial_a), "concurrent stream A corrupted"
assert torch.equal(out_b, serial_b), "concurrent stream B corrupted"
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_graph_capture_replay(dist_init, dtype):
"""The op must survive CUDA-graph capture: the replayed output must be
bit-identical to the eager output and stable across repeated replays."""
layer, q_int4, scales_gn = _build_layer(4096, 512, 4324, dtype)
wq, ws, wz = _prepare(layer, dtype, 4096, 512)
torch.manual_seed(4325)
x = torch.randn(1, 4096, device=device, dtype=dtype)
eager = _run_op(x, wq, wz, ws)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
_run_op(x, wq, wz, ws) # warm-up side streams/allocator on capture path
torch.accelerator.synchronize()
with torch.cuda.graph(graph):
captured = _run_op(x, wq, wz, ws)
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(captured, eager), "graph replay differs from eager"
graph.replay()
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(captured, eager), "repeated replay not repeatable"
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_multi_row_tile_bit_repeatable(dist_init, dtype):
"""Row tiling with ticket reuse: N % 16 == 8 bypasses the WMMA
dispatch (N % 16 != 0), so M=100 runs scalar across two 64-row tiles
(the second with gy > 1 y-blocks). Every tile's tickets must be armed,
consumed, and re-armed correctly for all outputs to be written exactly
once — and the result must be bit-repeatable."""
outs, ref = _outputs_and_ref(
M=100, K=4096, N=2056, seed=1250, dtype=dtype, repeats=20
)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
# ---------------------------------------------------------------------------
# F. Split-K geometry corners: clipped non-first z blocks, small split
# counts, and non-128 group sizes.
# ---------------------------------------------------------------------------
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("k,group", [(4224, 128), (2112, 32)])
def test_scalar_clipped_tail_splitk(dist_init, dtype, k, group):
"""K % 256 != 0 with z_count > 1: the LAST z block is clipped by end_k,
its group prefetch guard (`k + 32 < end_k`) is what keeps the raw
zero/scale loads in bounds, and the reduce tail loop runs (z=17 -> one
remainder slice; z=9 -> one batch + one). K=2112/G=32 also puts two
group switches inside the clipped tail block."""
outs, ref = _outputs_and_ref(M=1, K=k, N=512, seed=1251, dtype=dtype, group=group)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("group", [32, 64])
def test_scalar_small_group_matches_fp32_reference(dist_init, dtype, group):
"""Group sizes below 128 make the group constants change every 32-K
round (G=32) or three times per z block (G=64); the factored-dequant
correction must track every change. G=32 is also the production MoE
group size."""
outs, ref = _outputs_and_ref(
M=1, K=2048, N=512, seed=1252, dtype=dtype, group=group
)
_assert_repeatable(outs)
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
@gfx1100_only
@pytest.mark.parametrize("dtype", DTYPES)
def test_scalar_two_way_split_bit_repeatable(dist_init, dtype):
"""z_count == 2 (K=512): the minimal split-K configuration — the old
CAS bug's onset regime — and the pure-remainder reduce tail."""
outs, _ = _outputs_and_ref(
M=1, K=512, N=512, seed=1253, dtype=dtype, repeats=REPEATS
)
_assert_repeatable(outs)
@@ -1,87 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kernel-selection / gating tests for the ROCm RDNA3 W4A16 GPTQ kernel.
Verifies that ``choose_mp_linear_kernel`` resolves a supported W4A16 GPTQ
config to ``RDNA3W4A16LinearKernel`` on gfx1100 (it is registered ahead of
``TritonW4A16LinearKernel`` in the ROCm priority list), and that
``RDNA3W4A16LinearKernel.can_implement`` rejects the configs it does not
support so selection falls through to the next kernel.
Run `pytest tests/kernels/quantization/test_rdna3_w4a16_selection.py`.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
from vllm.model_executor.kernels.linear import ( # noqa: E402
choose_mp_linear_kernel,
)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
RDNA3W4A16LinearKernel,
)
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
from vllm.scalar_type import scalar_types # noqa: E402
WEIGHT_TYPE = scalar_types.uint4b8 # symmetric int4, bias = 8
# The kernel is only selectable when running on gfx1100 with the custom op
# compiled in; otherwise can_implement rejects and selection falls through.
gfx1100_only = pytest.mark.skipif(
not (
on_gfx1100()
and hasattr(torch.ops, "_rocm_C")
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
),
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
)
@gfx1100_only
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_selection_prefers_rdna3(dtype):
"""A supported W4A16 GPTQ config resolves to the RDNA3 kernel on gfx1100."""
config = MPLinearLayerConfig(
full_weight_shape=(1024, 256),
partition_weight_shape=(1024, 256),
weight_type=WEIGHT_TYPE,
act_type=dtype,
group_size=128,
zero_points=False,
)
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
@gfx1100_only
@pytest.mark.parametrize(
"weight_type,group_size,N,full_k,expected_ok",
[
(scalar_types.uint4b8, 128, 256, 1024, True), # nominal: supported
(scalar_types.uint4b8, -1, 256, 1024, False), # channelwise unsupported
(scalar_types.uint4b8, 128, 252, 1024, False), # N not a multiple of 8
(scalar_types.uint4b8, 96, 256, 1024, False), # group does not divide K
(scalar_types.uint8b128, 128, 256, 1024, False), # wrong quant type
],
ids=["ok", "channelwise", "bad_n", "group_ndiv_k", "wrong_qtype"],
)
def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
"""can_implement gates on quant type, group size, and N divisibility."""
config = MPLinearLayerConfig(
full_weight_shape=(full_k, N),
partition_weight_shape=(full_k, N),
weight_type=weight_type,
act_type=torch.float16,
group_size=group_size,
zero_points=False,
)
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
assert ok is expected_ok, reason
@@ -1,934 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the ROCm Hybrid W4A16 kernel (HIP skinny + Triton prefill).
Run `pytest tests/kernels/quantization/test_rdna_hybrid_w4a16.py`.
"""
import importlib
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.is_rocm():
pytest.skip("ROCm only", allow_module_level=True)
pytest.importorskip("triton")
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
device = "cuda"
hybrid_module = importlib.import_module(
"vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16"
)
RDNAHybridW4A16LinearKernel = hybrid_module.RDNAHybridW4A16LinearKernel
pack_int4_exllama_shuffle = hybrid_module.pack_int4_exllama_shuffle
SUPPORTED_GROUP_SIZES = hybrid_module.SUPPORTED_GROUP_SIZES
MAX_SKINNY_BATCH_SIZE = hybrid_module.MAX_SKINNY_BATCH_SIZE
LDS_CAPACITY_ELEMENTS = hybrid_module.LDS_CAPACITY_ELEMENTS
MEDIUM_SKINNY_LIMIT_ELEMENTS = hybrid_module.MEDIUM_SKINNY_LIMIT_ELEMENTS
# ---------------------------------------------------------------------------
# Reference implementation
# ---------------------------------------------------------------------------
def _pack_zp_rows_for_kernel(zp_nkg: torch.Tensor) -> torch.Tensor:
"""Pack raw uint4 zero points along N: [N, G] int32 -> [N//8, G] int32.
Row n's nibble lands in word[n//8] at bits 4*(n%8).
"""
assert zp_nkg.dtype == torch.int32
N, G = zp_nkg.shape
assert N % 8 == 0
shifts = (torch.arange(8, device=zp_nkg.device, dtype=torch.int32) * 4)[:, None]
return torch.sum(
(zp_nkg.view(N // 8, 8, G) & 0xF) << shifts, dim=1, dtype=torch.int32
).contiguous()
def _rdna_hybrid_w4a16_reference(
x_mk: torch.Tensor,
w_int4_nk: torch.Tensor,
scales_nkg: torch.Tensor,
zp_nkg: torch.Tensor | None,
group_size: int,
bias: torch.Tensor | None,
) -> torch.Tensor:
"""Reference for the Hybrid W4A16 op.
x_mk: [M, K] fp16/bf16
w_int4_nk: [N, K] int32 with raw uint4 values in [0, 15]
scales_nkg: [N, K//G] fp16/bf16
zp_nkg: [N, K//G] int32 raw zero points in [0, 15], or None for
symmetric (uint4b8, dequant subtracts 8)
"""
G = group_size
N, K = w_int4_nk.shape
assert K % G == 0
s_full = scales_nkg.repeat_interleave(G, dim=1).to(torch.float32) # [N, K]
if zp_nkg is None:
z_full = torch.full((N, K), 8.0, device=x_mk.device, dtype=torch.float32)
else:
z_full = zp_nkg.repeat_interleave(G, dim=1).to(torch.float32)
w_fp = (w_int4_nk.to(torch.float32) - z_full) * s_full # [N, K]
out = x_mk.to(torch.float32) @ w_fp.t() # [M, N]
if bias is not None:
out = out + bias.to(torch.float32)
return out.to(x_mk.dtype)
# ---------------------------------------------------------------------------
# Forward correctness: decode (HIP skinny) + prefill (Triton)
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize(
"M",
[1, MAX_SKINNY_BATCH_SIZE, MAX_SKINNY_BATCH_SIZE + 1, 64],
ids=["M=1_decode", "M=5_decode", "M=6_prefill", "M=64_prefill"],
)
def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M):
"""Smoke test the registered custom op for both decode and prefill batches.
Verifies the dispatch logic in `_rdna_hybrid_w4a16_apply_impl`:
- supported M and K*M: HIP wvSplitK_int4_g
- otherwise: Triton prefill kernel
"""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
set_random_seed(0)
K, N = 1024, 256
assert K % group_size == 0 and K % 8 == 0 and N % 8 == 0
# Activations.
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
# Weights as raw uint4 in [N, K], packed to ExLlama shuffle [N, K//8].
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
w_q_i32 = pack_int4_exllama_shuffle(w_int4_nk).contiguous() # [N, K//8] int32
w_q = w_q_i32.view(torch.int8) # same bytes viewed as int8 [N, K//2]
# Scales [N, K//G] in act dtype.
scales_nkg = (
0.05 * torch.rand((N, K // group_size), device=device, dtype=torch.float32)
).to(dtype)
# Optional raw zero points, [N, K//G] int32 for the reference and packed
# along N for the op.
if has_zp:
zp_nkg = torch.randint(
0, 16, (N, K // group_size), device=device, dtype=torch.int32
)
w_zp = _pack_zp_rows_for_kernel(zp_nkg)
else:
zp_nkg = None
w_zp = None
from vllm.utils.platform_utils import num_compute_units
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
x_mk,
w_q,
scales_nkg,
w_zp,
None, # bias
num_compute_units(),
group_size,
)
ref = _rdna_hybrid_w4a16_reference(
x_mk, w_int4_nk, scales_nkg, zp_nkg, group_size, bias=None
)
torch.testing.assert_close(out, ref, rtol=2e-2, atol=2e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
def test_rdna_hybrid_w4a16_apply_with_bias(dtype, M):
"""Bias is added correctly on both decode and prefill paths."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
set_random_seed(0)
K, N, G = 1024, 128, 128
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
w_q_i32 = pack_int4_exllama_shuffle(w_int4_nk).contiguous()
w_q = w_q_i32.view(torch.int8)
scales_nkg = (
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)
).to(dtype)
bias = torch.randn(N, device=device, dtype=dtype) * 0.1
from vllm.utils.platform_utils import num_compute_units
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
x_mk,
w_q,
scales_nkg,
None,
bias,
num_compute_units(),
G,
)
ref = _rdna_hybrid_w4a16_reference(x_mk, w_int4_nk, scales_nkg, None, G, bias=bias)
torch.testing.assert_close(out, ref, rtol=2e-2, atol=2e-2)
# ---------------------------------------------------------------------------
# pack_int4_exllama_shuffle round-trips correctly
# ---------------------------------------------------------------------------
def test_pack_int4_exllama_shuffle_layout():
"""Pack 8 K-values per int32 in interleave [0,2,4,6,1,3,5,7] order."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
set_random_seed(0)
N, K = 4, 16
w = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
packed = pack_int4_exllama_shuffle(w)
assert packed.shape == (N, K // 8) and packed.dtype == torch.int32
# Manual unshuffle using ExLlama shifts [0,16,4,20,8,24,12,28].
shifts = torch.tensor(
[0, 16, 4, 20, 8, 24, 12, 28], device=device, dtype=torch.int32
)
unshuffled = (packed.unsqueeze(-1) >> shifts) & 0xF
unshuffled = unshuffled.reshape(N, K)
torch.testing.assert_close(unshuffled, w)
# ---------------------------------------------------------------------------
# process_weights_after_loading: layout repack and zp normalization
# ---------------------------------------------------------------------------
def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor:
"""Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8]."""
assert w_int4_kn.dtype == torch.int32
K, N = w_int4_kn.shape
assert K % 8 == 0
out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device)
for i in range(8):
out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4)
return out.contiguous()
def _pack_int4_along_n_for_zp(zp_int4_gn: torch.Tensor) -> torch.Tensor:
"""Pack int4 zero points along N: [G, N] -> [G, N//8] int32 (CT layout)."""
assert zp_int4_gn.dtype == torch.int32
G, N = zp_int4_gn.shape
assert N % 8 == 0
shifts = torch.arange(8, device=zp_int4_gn.device, dtype=torch.int32) * 4
return torch.sum(
(zp_int4_gn.view(G, N // 8, 8) & 0xF) << shifts, dim=2, dtype=torch.int32
).contiguous()
def _build_dummy_layer(
w_ckpt_nk8: torch.Tensor,
scales_ckpt_nkg: torch.Tensor,
zeros_ckpt: torch.Tensor | None,
):
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
weight_loader = lambda *args, **kwargs: None
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"weight_packed",
PackedvLLMParameter(
data=w_ckpt_nk8,
weight_loader=weight_loader,
input_dim=1,
output_dim=0,
packed_factor=8,
packed_dim=1,
),
)
layer.register_parameter(
"weight_scale",
GroupQuantScaleParameter(
data=scales_ckpt_nkg,
weight_loader=weight_loader,
input_dim=1,
output_dim=0,
),
)
if zeros_ckpt is not None:
layer.register_parameter(
"weight_zero_point",
PackedColumnParameter(
data=zeros_ckpt,
weight_loader=weight_loader,
output_dim=0,
packed_factor=8,
packed_dim=0,
),
)
return layer
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_init):
"""uint4b8 (symmetric): w_q -> [N, K//8] int8 ExLlama shuffle, no zp param."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
set_random_seed(0)
K, N = 256, 128
G = group_size
assert K % G == 0
# Reference unpacked weights, then pack into CT checkpoint layout [N, K//8].
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn)
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
layer = _build_dummy_layer(w_ckpt_nk8, scales_ckpt_nkg, zeros_ckpt=None)
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=G,
zero_points=False,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
)
kernel.process_weights_after_loading(layer)
# Skinny weight is stored once as int8 [N, K//2]; the Triton path
# reinterprets it as int32 [N, K//8] via a view (no separate parameter).
assert layer.weight_packed.dtype == torch.int8
assert tuple(layer.weight_packed.shape) == (N, K // 2)
w_q_i32 = layer.weight_packed.view(torch.int32)
assert tuple(w_q_i32.shape) == (N, K // 8)
expected_packed = pack_int4_exllama_shuffle(w_int4_kn.t().contiguous())
torch.testing.assert_close(w_q_i32, expected_packed)
# Scales: [N, K//G] (skinny layout, no transpose since CT already had it).
assert tuple(layer.weight_scale.shape) == (N, K // G)
torch.testing.assert_close(layer.weight_scale, scales_ckpt_nkg)
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_init):
"""uint4 (asymmetric): zero points are packed [N//8, K//G] int32."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
set_random_seed(0)
K, N = 256, 128
G = group_size
assert K % G == 0 and N % 8 == 0
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn)
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
# CT zero-point layout is N-packed: [N//8, K//G] int32.
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
zeros_packed_gn8 = _pack_int4_along_n_for_zp(zeros_int4_gn) # [K//G, N//8]
zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G]
layer = _build_dummy_layer(w_ckpt_nk8, scales_ckpt_nkg, zeros_ckpt=zeros_ckpt_n8kg)
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4,
act_type=torch.float16,
group_size=G,
zero_points=True,
)
kernel = RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
kernel.process_weights_after_loading(layer)
# Zero-points: [N//8, K//G] int32, row n's nibble at word[n//8]
# bits 4*(n%8).
assert layer.weight_zero_point.dtype == torch.int32
assert tuple(layer.weight_zero_point.shape) == (N // 8, K // G)
expected_zp = _pack_zp_rows_for_kernel(zeros_int4_gn.t().contiguous())
torch.testing.assert_close(layer.weight_zero_point, expected_zp)
# Quantized weights match symmetric path's layout regardless of zp.
w_q_i32 = layer.weight_packed.view(torch.int32)
assert tuple(w_q_i32.shape) == (N, K // 8)
expected_packed = pack_int4_exllama_shuffle(w_int4_kn.t().contiguous())
torch.testing.assert_close(w_q_i32, expected_packed)
# ---------------------------------------------------------------------------
# can_implement enforces the supported-group-size policy
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize(
"group_size,expected_ok", [(32, True), (64, True), (128, True), (256, False)]
)
def test_hybrid_can_implement_group_size(group_size, expected_ok):
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
K, N = 1024, 256
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=group_size,
zero_points=False,
)
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
assert ok is expected_ok
# ---------------------------------------------------------------------------
# Tests for the HIP wvSplitK_int4_g decode kernel
# ---------------------------------------------------------------------------
def _hip_skinny_reference(
a_mk: torch.Tensor,
w_int4_nk: torch.Tensor,
scales_nkg: torch.Tensor,
*,
group_size: int,
zp_nkg: torch.Tensor | None,
) -> torch.Tensor:
"""Reference for HIP skinny: C = A @ (W - zp) * S.
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
None for symmetric uint4b8, where the kernel subtracts a constant 8.
"""
K = a_mk.shape[1]
N = w_int4_nk.shape[0]
num_groups = K // group_size
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
s = scales_nkg.to(torch.float32).unsqueeze(-1)
w_dequant = ((w_g - zp) * s).view(N, K)
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize(
"M,K,N,G",
[
(1, 256, 256, 32),
(1, 256, 256, 64),
(1, 512, 256, 128),
(2, 512, 256, 64),
(3, 256, 512, 64),
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
(4, 512, 256, 32),
(4, 256, 512, 64),
(5, 512, 256, 128),
(5, 256, 256, 32),
# N == 8 is exactly one packed zero-point word.
(1, 256, 8, 32),
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
# unrolled block; every K above is below that bound, so without these
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
(1, 4096, 256, 128),
(2, 2560, 256, 32),
(3, 4096, 256, 64),
(4, 2560, 256, 128),
(5, 4096, 256, 32),
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
# tight case there: each lane's A_CHUNK spans exactly one scale group.
(1, 4096, 256, 32),
(1, 8192, 256, 128),
(1, 16384, 256, 64),
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
# walks K in chunks; elsewhere the op rejects these, as the layer
# routes them to Triton. The per-row window shrinks as the batch grows,
# so these cover a single reload and several, with and without a short
# final chunk. 20992 is not a multiple of the K step either, so it
# crosses a chunk boundary *and* has a ragged K tail.
(2, 20480, 512, 64),
(3, 16384, 512, 32),
(4, 16384, 512, 128),
(5, 21504, 512, 32),
(5, 20992, 512, 128),
],
)
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
pytest.skip("deep-K HIP path is gfx115x only")
set_random_seed(0)
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
b_packed_i32 = pack_int4_exllama_shuffle(w_int4_nk)
b_packed_i8 = b_packed_i32.view(torch.int8)
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
dtype
)
zp_nkg = None
zp_packed = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
cu_count = num_compute_units()
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("G", [32, 128])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
"""Scale and zero-point rows may be padded past K // group_size.
Both paths index them by their own row stride, so a padded row is read
correctly rather than bleeding into the next row. M picks the path: the
HIP skinny kernel at 2, the Triton prefill kernel at 64.
"""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
K, N = 2048, 256
num_groups = K // G
pad = 7
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
scales_padded[:, :num_groups] = scales
scales_view = scales_padded[:, :num_groups]
assert scales_view.stride(0) == num_groups + pad
zp_nkg = zp_view = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
zp_padded = torch.zeros(
(N // 8, num_groups + pad), device=device, dtype=torch.int32
)
zp_padded[:, :num_groups] = zp_packed
zp_view = zp_padded[:, :num_groups]
cu_count = num_compute_units()
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
# ---------------------------------------------------------------------------
# Tests for the full hybrid dispatch (HIP decode + Triton prefill)
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"M,K,N,G",
[
(1, 256, 256, 64),
(1, 512, 256, 32),
(1, 512, 256, 128),
# Past LDS: HIP on gfx115x, Triton elsewhere.
(2, 20480, 256, 64),
(32, 512, 256, 64),
(64, 1024, 256, 128),
],
)
def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
"""Test the full hybrid dispatch via the custom op."""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
b_packed_i32 = pack_int4_exllama_shuffle(w_int4_nk)
b_packed_i8 = b_packed_i32.view(torch.int8)
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
dtype
)
cu_count = num_compute_units()
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, b_packed_i8, scales, None, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.skipif(
not hasattr(torch.ops, "_rocm_C")
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
reason="wvSplitK_int4_g not built",
)
def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
"""Act-dtype zero points raise instead of being read as packed words.
Both layouts are 2D with compatible extents, so only the dtype separates
them; misreading one as the other returns wrong numbers silently.
"""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
K, N, G, M = 256, 64, 128, 1
a = torch.randn((M, K), device=device, dtype=torch.float16)
w = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
torch.int8
)
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
zp_unpacked = torch.zeros((N, K // G), device=device, dtype=torch.float16)
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.skipif(
not hasattr(torch.ops, "_rocm_C")
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
reason="wvSplitK_int4_g not built",
)
def test_wvsplitk_int4_g_rejects_undersized_weight_view():
"""A sufficient row stride does not prove the row holds K/2 packed bytes.
Row padding makes stride(0) > K/2 legitimate, but the logical size still
has to be checked: this view has a valid stride and half the required
bytes, and without the size check the kernel reads past it.
"""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
K, N, G, M = 512, 256, 128, 1
a = torch.randn((M, K), device=device, dtype=torch.float16)
# Full-size backing allocation, so a missing check misreads rather than
# running off the end of the buffer.
backing = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
torch.int8
)
undersized = backing[:, : K // 4]
assert undersized.stride(0) == K // 2
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
with pytest.raises(RuntimeError, match=r"must contain M\*K/2 bytes"):
ops.wvSplitK_int4_g(undersized, a, scales, num_compute_units(), G, None, None)
# ---------------------------------------------------------------------------
# gfx11 weight row-stride padding
# ---------------------------------------------------------------------------
_weight_pad_bytes = hybrid_module._weight_pad_bytes
_act_pad_bytes = hybrid_module._act_pad_bytes
_WEIGHT_CLIFF_BYTES = hybrid_module._WEIGHT_CLIFF_BYTES
_ACT_CLIFF_BYTES = hybrid_module._ACT_CLIFF_BYTES
_STRIDE_PAD_BYTES = hybrid_module._STRIDE_PAD_BYTES
def test_weight_pad_bytes_only_moves_strides_on_the_cliff():
"""Pad 1024 B multiples, leave everything else dense."""
# On the cliff (K % 2048 == 0): 1024 B multiples.
for row_bytes in (1024, 2048, 4096, 5120, 6144, 7168, 8192):
assert _weight_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
# Off it, including strides that are 512 B multiples but not 1024 B ones:
# padding those measured as a real loss on gfx1151 (-14% at M=1 on a
# 4864 B row).
for row_bytes in (1280, 1536, 2560, 4864, 9472, 12800):
assert _weight_pad_bytes(row_bytes) == 0
# The padded stride is never back on the cliff.
for row_bytes in range(16, 16384, 16):
padded = row_bytes + _weight_pad_bytes(row_bytes)
assert padded % _WEIGHT_CLIFF_BYTES != 0 or row_bytes % _WEIGHT_CLIFF_BYTES
def test_act_pad_bytes_only_moves_strides_on_the_cliff():
"""Activations sit on a wider cliff than the packed weight: 2048 B."""
# On the cliff (K % 1024 == 0 for a 2-byte dtype): 2048 B multiples.
for row_bytes in (2048, 4096, 8192, 16384, 20480, 24576, 43008):
assert _act_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
# Off it, including 1024 B multiples that are not 2048 B ones -- the
# weight cliff must not be applied to activations.
for row_bytes in (1024, 3072, 5120, 10752, 19456, 37888):
assert _act_pad_bytes(row_bytes) == 0
# The padded stride is never back on the cliff.
for row_bytes in range(16, 65536, 16):
padded = row_bytes + _act_pad_bytes(row_bytes)
assert padded % _ACT_CLIFF_BYTES != 0 or row_bytes % _ACT_CLIFF_BYTES
def _row_padded_copy(t: torch.Tensor, pad_cols: int) -> torch.Tensor:
"""Copy of ``t`` whose stride(0) is ``t.shape[1] + pad_cols``."""
rows, cols = t.shape
buf = torch.empty((rows, cols + pad_cols), dtype=t.dtype, device=t.device)
buf[:, :cols].copy_(t)
return buf[:, :cols]
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
def test_weight_row_padding_does_not_change_results(dtype, has_zp, M):
"""Padding the weight rows is a pure layout change.
Both the HIP skinny kernel (M small) and the Triton prefill kernel take the
weight row stride from the tensor, so a padded layout must produce exactly
the same numbers as the dense one.
"""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
K, N, G = 512, 256, 128
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
w_q = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
dtype
)
zp = (
_pack_zp_rows_for_kernel(
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
)
if has_zp
else None
)
# 32 int32 columns = 128 B of pad, the production pad size.
w_q_pad = _row_padded_copy(w_q.view(torch.int32), 32).view(torch.int8)
assert w_q_pad.stride(0) == w_q.stride(0) + _STRIDE_PAD_BYTES
cu_count = num_compute_units()
dense = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, w_q, scales, zp, None, cu_count, G
)
padded = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, w_q_pad, scales, zp, None, cu_count, G
)
torch.testing.assert_close(padded, dense, rtol=0, atol=0)
@pytest.mark.skipif(
not hybrid_module._on_gfx1151(),
reason="row-stride padding is only enabled on gfx1151",
)
@pytest.mark.parametrize(
"K,expected_weight_stride",
[
(512, 256), # 256 B row: off the cliff, stored dense
(8192, 4096 + 128), # 4096 B row: on the cliff, padded
],
)
def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
"""The stored weight row stride follows the padding rule."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
set_random_seed(0)
N, G = 128, 128
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
layer = _build_dummy_layer(
_pack_int4_along_k_to_ckpt(w_int4_kn),
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16),
zeros_ckpt=None,
)
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=G,
zero_points=False,
)
RDNAHybridW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name=None,
).process_weights_after_loading(layer)
# Weight rows: stride is in int8 elements, i.e. bytes.
assert layer.weight_packed.stride(0) == expected_weight_stride
# The int32 view the Triton path uses survives the padded stride.
assert tuple(layer.weight_packed.view(torch.int32).shape) == (N, K // 8)
# Metadata is untouched by this change.
assert layer.weight_scale.is_contiguous()
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize(
"M,K,gfx115x,expected_path",
[
(4, 8192, False, "hip"),
(4, 9728, False, "hip"),
(4, 9856, False, "triton"),
(4, 9856, True, "hip"),
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
],
ids=[
"regular_limit",
"medium_range",
"above_medium",
"above_medium_gfx115x",
"batch_too_large",
],
)
def test_rdna_hybrid_w4a16_dispatch_boundaries(
M, K, gfx115x, expected_path, monkeypatch
):
"""Route only supported regular and medium skinny shapes to HIP.
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
HIP there; the batch bound holds on every target.
"""
import vllm._custom_ops as ops
N, G = 16, 128
x_mk = torch.empty((M, K), dtype=torch.float16)
w_q = torch.empty((N, K // 2), dtype=torch.int8)
scales = torch.empty((N, K // G), dtype=torch.float16)
called = []
def fake_hip(*args, **kwargs):
called.append("hip")
return torch.empty((M, N), dtype=x_mk.dtype)
def fake_triton(*args, **kwargs):
called.append("triton")
return torch.empty((M, N), dtype=x_mk.dtype)
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
x_mk, w_q, scales, None, None, 40, G
)
assert output.shape == (M, N)
assert called == [expected_path]
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
def test_rdna_hybrid_w4a16_medium_skinny_matches_reference(dtype, has_zp):
"""Validate the HIP medium path against an FP32 dequantization reference."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
M, K, N, G = 4, 9728, 256, 128
assert LDS_CAPACITY_ELEMENTS < K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
w_q = pack_int4_exllama_shuffle(w_int4_nk).contiguous().view(torch.int8)
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
dtype
)
zp = (
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
if has_zp
else None
)
w_zp = _pack_zp_rows_for_kernel(zp) if zp is not None else None
output = torch.ops.vllm.rdna_hybrid_w4a16_apply(
x_mk, w_q, scales, w_zp, None, num_compute_units(), G
)
reference = _rdna_hybrid_w4a16_reference(x_mk, w_int4_nk, scales, zp, G, bias=None)
torch.testing.assert_close(output, reference, rtol=2e-2, atol=2e-2)
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# This is a test for the AITER group_fp8_quant op.
# It tests if the AITER op is
# 1. correctly defined the relationship between
# implementation and fake function
# 2. can be used with torch.compile
# 3. can be used with CUDA graphs
# This file will be skipped if AITER is not installed
# and the platform is not ROCm.
import importlib.util
import pytest
import torch
# this import statement is needed to ensure the ops are registered
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import current_platform
# Check if aiter package is installed
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = pytest.mark.skipif(
not (current_platform.is_rocm() and aiter_available),
reason="AITER ops are only available on ROCm with aiter package installed",
)
def test_rocm_aiter_group_fp8_quant_fake_implementation():
"""Test that the fake implementation is correctly
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant."""
# Create test tensors
M = 128
N = 4096
group_size = 128
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
# Verify the op's fake implementation using torch.library.opcheck
# This checks that the fake function returns tensors with correct shapes and dtypes
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_group_fp8_quant,
(input_tensor, group_size),
test_utils=("test_faketensor",),
)
def test_rocm_aiter_group_fp8_quant_torch_compile_with_cudagraph():
"""Test that rocm_aiter_ops.group_fp8_quant
with group size 128 can be used with
torch.compile in cudagraph mode."""
# Create test tensors
M = 128
N = 4096
group_size = 128
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
# Define a function that uses the op
def group_fp8_quant_fn(x):
return rocm_aiter_ops.group_fp8_quant(x, group_size)
# Compile with cudagraph mode
compiled_fn = torch.compile(
group_fp8_quant_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
# Run eager mode
x_fp8_eager, scales_eager = group_fp8_quant_fn(input_tensor)
# Run compiled version (first run will trigger compilation)
x_fp8_compiled, scales_compiled = compiled_fn(input_tensor)
# Verify shapes match
assert x_fp8_compiled.shape == x_fp8_eager.shape
assert scales_compiled.shape == scales_eager.shape
# Verify expected shapes
assert x_fp8_compiled.shape == (M, N)
expected_scale_cols = (N + group_size - 1) // group_size
assert scales_compiled.shape == (M, expected_scale_cols)
# Verify results match
assert torch.allclose(
x_fp8_compiled.to(torch.float32),
x_fp8_eager.to(torch.float32),
rtol=1e-2,
atol=1e-2,
)
assert torch.allclose(scales_compiled, scales_eager, rtol=1e-3, atol=1e-3)
# Test with different input (reusing compiled graph)
input_tensor_2 = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
x_fp8_eager_2, scales_eager_2 = group_fp8_quant_fn(input_tensor_2)
x_fp8_compiled_2, scales_compiled_2 = compiled_fn(input_tensor_2)
# Verify second run also produces correct results
assert torch.allclose(
x_fp8_compiled_2.to(torch.float32),
x_fp8_eager_2.to(torch.float32),
rtol=1e-2,
atol=1e-2,
)
assert torch.allclose(scales_compiled_2, scales_eager_2, rtol=1e-3, atol=1e-3)
def test_rocm_aiter_group_fp8_quant_different_shapes():
"""Test rocm_aiter_ops.group_fp8_quant with different input shapes."""
group_size = 128
test_shapes = [
(64, 2048),
(256, 8192),
(32, 1024),
(512, 4096),
]
for M, N in test_shapes:
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
x_fp8, scales = rocm_aiter_ops.group_fp8_quant(input_tensor, group_size)
# Verify shapes
assert x_fp8.shape == (M, N)
expected_scale_cols = (N + group_size - 1) // group_size
assert scales.shape == (M, expected_scale_cols)
# Verify dtypes
from aiter import dtypes
assert x_fp8.dtype == dtypes.fp8
assert scales.dtype == torch.float32
@@ -1,34 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end smoke test for CT W4A16 models on ROCm.
This validates that a real compressed-tensors W4A16 model can run inference
end-to-end (which will exercise the Triton W4A16 kernel when selected).
Run `pytest tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py`.
"""
import pytest
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"model_path",
[
"nm-testing/tinyllama-oneshot-w4a16-group128-v2",
],
)
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm")
def test_rocm_compressed_tensors_w4a16_e2e(
vllm_runner, example_prompts, model_path, max_tokens
):
# Use fp16 activations for maximum compatibility.
# gpu_memory_utilization lowered to work on shared nodes.
with vllm_runner(
model_path, dtype="float16", gpu_memory_utilization=0.3
) as vllm_model:
# If the W4A16 kernel is broken, this will typically throw.
vllm_model.generate_greedy(example_prompts, max_tokens=max_tokens)
@@ -1,729 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm AITER FP4 and MXFP4 tests.
This file keeps the ROCm-specific FP4 coverage:
- ROCm AITER FP4 env and enablement gates
- large-shape MXFP4 wrapper checks through vLLM's public helper op
- AITER Triton MXFP4 quant format and determinism
- FP4 GEMM, preshuffled-scale, hardware-quant, and skinny decode
paths
Generic MXFP4 wrapper/reference checks already live in
``tests/kernels/quantization/test_quark.py``.
"""
import importlib
import warnings
import pytest
import torch
from tests.kernels.utils import _assert_accurate, _assert_deterministic
from tests.quantization.reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
LLAMA_SHAPES = [
(64, 4096),
(64, 11008),
(32, 14336),
]
GEMM_ATOL = 0.5
GEMM_RTOL = 0.0
GEMM_MAX_VIOLATION_FACTOR = 2.0
A4W4_GEMM_PASS_RATES = {
(64, 128, 64): 0.9999,
(128, 256, 128): 0.9999,
(128, 4096, 4096): 0.9999,
(256, 4096, 11008): 0.9999,
(64, 8192, 28672): 0.99765,
}
SKINNY_GEMM_PASS_RATES = {
(4096, 4096): 0.9999,
(4096, 11008): 0.99235,
(8192, 8192): 0.99765,
}
PRESHUFFLED_SHAPES = [
(64, 4096, 8192),
(32, 16384, 16384),
]
def _reload_envs():
import vllm.envs as envs
return importlib.reload(envs)
@pytest.fixture(autouse=True)
def _restore_rocm_env_state():
"""Restore global env + AITER flag state after every test."""
yield
_reload_envs()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
def _assert_aiter_supported() -> None:
from vllm._aiter_ops import is_aiter_found_and_supported
assert is_aiter_found_and_supported(), (
"aiter is required on supported ROCm hardware for this test"
)
def _format_observed_rate(count: int, total: int) -> str:
return f"{count / total:.4%} ({count}/{total})"
def _format_allowed_rate(rate: float, total: int) -> str:
allowed_count = int(rate * total)
return f"{rate:.4%} (<= {allowed_count}/{total})"
def _print_close_stats(
label: str,
actual: torch.Tensor,
expected: torch.Tensor,
*,
atol: float,
rtol: float,
pass_rate: float = 1.0,
max_atol: float | None = None,
) -> None:
abs_diff = (actual - expected).abs().float().flatten()
expected_abs = expected.abs().float().flatten()
allowed = atol + rtol * expected_abs
within = abs_diff <= allowed
total = abs_diff.numel()
passed = int(within.sum().item())
failed = total - passed
allowed_fail_rate = 1.0 - pass_rate
max_abs = abs_diff.max().item()
mean_abs = abs_diff.mean().item()
p99_abs = torch.quantile(abs_diff, 0.99).item()
p999_abs = torch.quantile(abs_diff, 0.999).item()
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
msg = (
"[rocm_aiter_fp4] "
f"{label}: "
f"pass={passed / total:.4%} ({passed}/{total}) "
f"fail={_format_observed_rate(failed, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"atol={atol:g} "
f"rtol={rtol:g} "
)
if max_atol is not None:
above_max_count = int((abs_diff > max_atol).sum().item())
msg += (
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
)
msg += (
f"max_abs={max_abs:.6g} "
f"mean_abs={mean_abs:.6g} "
f"p99_abs={p99_abs:.6g} "
f"p999_abs={p999_abs:.6g} "
f"worst_ratio={worst_ratio:.6g}"
)
print(msg)
if failed > 0:
warnings.warn(msg, stacklevel=2)
# Env and enablement tests ------------------------------------------------
def test_fp4_env_defaults():
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
@pytest.mark.parametrize(
("use_aiter", "use_fp4bmm"),
[(True, True), (True, False), (False, True)],
)
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
use_aiter,
use_fp4bmm,
monkeypatch,
):
"""The ROCm FP4 AITER gates should depend only on the env toggles and the
gfx950 hardware check.
Expected values are derived from the *actual* platform rather than a patched
arch (gfx950 -> MI355, gfx942 -> MI300). A failure in this group therefore
points straight at the hardware it ran on instead of a mocked value.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
on_gfx950_value = on_gfx950()
expected_asm_gemm = use_aiter and on_gfx950_value
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
# Large-shape MXFP4 wrapper tests -----------------------------------------
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
reference even on larger Llama-like shapes."""
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
rows, cols = shape
x = torch.randn(rows, cols, dtype=dtype)
out = quant_dequant_mxfp4(x)
ref = qdq_mxfp4_torch(x, "even")
assert out.shape == x.shape
assert out.dtype == dtype
assert torch.isfinite(out).all()
torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0)
def test_vllm_quant_dequant_mxfp4_is_deterministic():
"""The public vLLM MXFP4 QDQ helper should stay bitwise deterministic."""
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(3)
x = torch.randn(64, 128, dtype=torch.bfloat16)
_assert_deterministic(quant_dequant_mxfp4, x, n_runs=4)
# Triton MXFP4 quant tests ------------------------------------------------
def test_aiter_dynamic_mxfp4_quant_output_format():
"""dynamic_mxfp4_quant returns packed uint8 FP4 values and E8M0 uint8 scales.
OCP MXFP4: block_size=32, 2 FP4 E2M1 values packed per byte.
Scale shape: (M, K // 32); one E8M0 exponent byte per 32-element block.
"""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
M, K = 64, 256
x = torch.randn(M, K, dtype=torch.bfloat16)
x_fp4, x_scale = dynamic_mxfp4_quant(x)
# FP4 values packed 2-per-byte; shape (M, K // 2)
assert x_fp4.shape == (M, K // 2), (
f"Expected fp4 shape ({M}, {K // 2}), got {x_fp4.shape}"
)
assert x_fp4.dtype == torch.uint8
# One E8M0 scale byte per 32-element block; shape (M, K // 32)
assert x_scale.shape == (M, K // 32), (
f"Expected scale shape ({M}, {K // 32}), got {x_scale.shape}"
)
assert x_scale.dtype == torch.uint8
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
"shape",
[
(128, 4096), # Llama-7B hidden
(256, 11008), # Llama-7B FFN
(64, 14336), # Llama-70B FFN
(512, 8192), # Llama-70B hidden
(32, 28672), # DeepSeek-style large FFN
],
)
def test_aiter_dynamic_mxfp4_quant_llama_shapes(shape, dtype):
"""MXFP4 quantization output format is correct for Llama-class weight shapes.
Targets B200 parity: NVIDIA nvfp4 tests parametrize over Llama shapes
(7168, 14336, 28672...). This covers gfx950 MXFP4 quant format validation.
"""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
M, K = shape
x = torch.randn(M, K, dtype=dtype)
x_fp4, x_scale = dynamic_mxfp4_quant(x)
assert x_fp4.shape == (M, K // 2)
assert x_fp4.dtype == torch.uint8
assert x_scale.shape == (M, K // 32)
assert x_scale.dtype == torch.uint8
assert not torch.any(torch.isnan(x_fp4.float()))
def test_aiter_dynamic_mxfp4_quant_determinism():
"""dynamic_mxfp4_quant is bitwise deterministic across repeated runs."""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(7)
x = torch.randn(128, 256, dtype=torch.bfloat16)
_assert_deterministic(dynamic_mxfp4_quant, x, n_runs=4)
# gfx950 hardware FP4 GEMM tests ------------------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
[
(64, 128, 64), # small square-ish
(128, 256, 128), # medium
(128, 4096, 4096), # Llama-7B hidden square
(256, 4096, 11008), # Llama-7B FFN
(64, 8192, 28672), # Llama-70B FFN
],
)
def test_aiter_fp4_gemm_a4w4_accuracy(shape):
"""AITER A4W4 FP4 GEMM output is close to matmul on dequantized weights.
Tests gfx950 parity with B200 test_nvfp4_scaled_mm.py: E2M1 x E2M1 GEMM
with block-scaled quantization, multiple Llama-class shapes.
Requires gfx950 hardware; gracefully skips elsewhere.
"""
atol = GEMM_ATOL
rtol = GEMM_RTOL
pass_rate = A4W4_GEMM_PASS_RATES[shape]
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
M, K, N = shape
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16) # weight stored as (N, K)
# Quantize
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
# FP4 GEMM computes A_fp4 (M, K) @ B_fp4.T (K, N) into (M, N).
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
assert out.shape == (M, N), f"Expected ({M}, {N}), got {out.shape}"
assert out.dtype == torch.bfloat16
assert not torch.any(torch.isnan(out))
# Reference: matmul on dequantized FP4 inputs (quark roundtrip)
A_dq = quant_dequant_mxfp4(A)
B_dq = quant_dequant_mxfp4(B)
ref = torch.matmul(A_dq.float(), B_dq.t().float())
# FP4 has only 3 mantissa bits; large K amplifies the accumulation error.
# These budgets were measured on gfx950 and keep p99/p999 drift visible
# without pretending the kernel is close to BF16 matmul everywhere.
_print_close_stats(
f"a4w4_gemm shape={shape}",
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
)
_assert_accurate(
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
PRESHUFFLED_SHAPES,
)
def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
"""The preshuffled FP4 GEMM path should run on tuned shapes with the same
packed inputs vLLM uses in production.
This kernel only advertises a narrow tuned `(N, K)` set on gfx950. The
important contract here is that those tuned shapes execute with the
production-style shuffled weights and stay deterministic.
"""
_assert_aiter_supported()
from aiter import per_1x32_f4_quant_hip
from aiter.ops.shuffle import shuffle_weight
from aiter.ops.triton.gemm_afp4wfp4 import (
gemm_afp4wfp4_preshuffle,
)
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm._aiter_ops import rocm_aiter_ops
torch.set_default_device("cuda")
torch.manual_seed(1)
M, K, N = shape
assert M <= 64
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
# must agree with the config the kernel itself resolves.
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
A_q, A_s = per_1x32_f4_quant_hip(A, shuffle=M >= 32)
scale_rows, scale_cols = B_scale.shape
B_scale = (
B_scale.view(scale_rows // 32, 2, 16, scale_cols // 8, 2, 4, 1)
.permute(0, 3, 5, 2, 4, 1, 6)
.contiguous()
.view(scale_rows, scale_cols)
)
B_fp4 = shuffle_weight(B_fp4, layout=(16, 16))
if M >= 32:
A_s = A_s.contiguous().view(torch.uint8).reshape(A_s.shape[0] // 32, -1)
else:
A_s = A_s[:M, ...].contiguous().view(torch.uint8)
B_scale = B_scale.contiguous().view(torch.uint8).reshape(B_scale.shape[0] // 32, -1)
def run_preshuffled() -> torch.Tensor:
y = torch.empty(M, N, device="cuda", dtype=torch.bfloat16)
return gemm_afp4wfp4_preshuffle(
A_q.contiguous().view(torch.uint8),
B_fp4.contiguous().view(torch.uint8).reshape(B_fp4.shape[0] // 16, -1),
A_s,
B_scale,
torch.bfloat16,
y,
)
out = run_preshuffled()
assert out.shape == (M, N)
assert out.dtype == torch.bfloat16
assert torch.isfinite(out).all()
_assert_deterministic(run_preshuffled, n_runs=3)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
has no tuned preshuffled config even though twice its K does."""
_assert_aiter_supported()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_gemm_a4w4_determinism():
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
Targets parity with B200 FP4 determinism requirements.
Requires gfx950.
"""
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(2)
M, K, N = 128, 256, 128
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
def run_gemm():
return gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
_assert_deterministic(run_gemm, n_runs=4)
# gfx950 hardware FP4 dynamic quantization --------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
[
(128, 256),
(256, 4096),
(64, 14336),
],
)
def test_aiter_hardware_fp4_dynamic_quant_format(shape):
"""Aiter hardware FP4 dynamic quant produces correct output format.
Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1).
Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output.
Requires gfx950.
"""
_assert_aiter_supported()
from aiter import dynamic_per_group_scaled_quant_fp4
torch.set_default_device("cuda")
M, K = shape
group_size = 32
x = torch.randn(M, K, dtype=torch.bfloat16)
out_fp4 = torch.empty(M, K // 2, dtype=torch.uint8)
scales = torch.empty(M, K // group_size, dtype=torch.uint8)
dynamic_per_group_scaled_quant_fp4(out_fp4, x, scales, group_size)
assert out_fp4.shape == (M, K // 2), (
f"Shape {shape}: expected fp4 ({M}, {K // 2}), got {out_fp4.shape}"
)
assert scales.shape == (M, K // group_size), (
f"Shape {shape}: expected scale ({M}, {K // group_size}), got {scales.shape}"
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_hardware_fp4_quant_vs_triton():
"""Hardware FP4 quant should dequantize back to the same MXFP4 values as
the Triton path.
The packed bytes are not identical on gfx950, but with
``shuffle_scale=False`` the dequantized results match tightly.
"""
_assert_aiter_supported()
from aiter import dynamic_per_group_scaled_quant_fp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(5)
M, K = 128, 256
group_size = 32
x = torch.randn(M, K, dtype=torch.bfloat16)
# Hardware path
out_hw = torch.empty(M, K // 2, dtype=torch.uint8)
scales_hw = torch.empty(M, K // group_size, dtype=torch.uint8)
dynamic_per_group_scaled_quant_fp4(
out_hw, x, scales_hw, group_size, shuffle_scale=False
)
# Triton path
out_triton, scales_triton = dynamic_mxfp4_quant(x)
assert out_hw.shape == out_triton.shape
assert scales_hw.shape == scales_triton.shape
dq_hw = dq_mxfp4_torch(out_hw, scales_hw, torch.bfloat16)
dq_triton = dq_mxfp4_torch(out_triton, scales_triton, torch.bfloat16)
_print_close_stats(
"hardware_quant_vs_triton",
dq_hw.float(),
dq_triton.float(),
atol=0.25,
rtol=0.0,
)
torch.testing.assert_close(dq_hw, dq_triton, atol=0.25, rtol=0.0)
# ROCm skinny GEMM FP4 tests ----------------------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize("M", [1, 2, 4, 8]) # decode / skinny batch sizes
@pytest.mark.parametrize("N, K", [(4096, 4096), (4096, 11008), (8192, 8192)])
def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
"""FP4 GEMM accuracy for skinny (small-M) shapes (decode phase).
Skinny GEMMs (M=1..8) are the bottleneck in decode phase.
Tests parity with B200 nvfp4_scaled_mm tests at decode batch sizes.
Requires gfx950.
"""
atol = GEMM_ATOL
rtol = GEMM_RTOL
pass_rate = SKINNY_GEMM_PASS_RATES[(N, K)]
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
assert out.shape == (M, N)
assert out.dtype == torch.bfloat16
assert not torch.any(torch.isnan(out))
# Accumulate in FP32, then match the GEMM's BF16 output rounding.
A_dq = quant_dequant_mxfp4(A)
B_dq = quant_dequant_mxfp4(B)
ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float()
_print_close_stats(
f"skinny_gemm M={M} N={N} K={K}",
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
)
_assert_accurate(
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
("shape", "supported"),
[
((2560, 80), True), # shared_expert gate/up_proj (in=2560 -> sn=80)
((1280, 80), True), # gate/up_proj under TP=2 (out sharded)
((2560, 8), True), # minimal aligned columns
((32, 8), True), # minimal aligned rows and columns
((2560, 20), False), # shared_expert down_proj (in=640 -> sn=20)
((2560, 10), False), # down_proj under TP=2 (in 640->320 -> sn=10)
((30, 8), False), # rows not a multiple of 32
],
)
def test_asm_fp4_scale_swizzle_supported_shape_rules(shape, supported):
"""The ASM swizzle needs rows % 32 == 0 and columns % 8 == 0."""
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
_asm_fp4_scale_swizzle_supported,
)
weight_scale = torch.empty(shape, dtype=torch.uint8, device="cpu")
assert _asm_fp4_scale_swizzle_supported(weight_scale) is supported
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_asm_fp4_scale_swizzle_rejects_non_2d():
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
_asm_fp4_scale_swizzle_supported,
)
assert not _asm_fp4_scale_swizzle_supported(
torch.empty(2560, dtype=torch.uint8, device="cpu")
)
assert not _asm_fp4_scale_swizzle_supported(
torch.empty(80, 2, 16, dtype=torch.uint8, device="cpu")
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
[
(True, True, (1280, 4096), (1280, 256), True),
(True, False, (1280, 4096), (1280, 256), False),
(False, True, (1280, 4096), (1280, 256), False),
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
(True, True, (2560, 160), (2560, 10), False),
],
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
)
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
):
"""A layer takes the ASM layout only when aiter has tuned configs for both
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
Otherwise it stays on the Triton layout."""
from torch.nn.parameter import Parameter
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
monkeypatch.setattr(
rocm_aiter_ops,
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
lambda n, k: preshuffle_tuned,
)
kernel = object.__new__(AiterMxfp4LinearKernel)
kernel.asm_gemm_supported = True
kernel.use_asm_gemm = False
kernel.out_dtype = torch.bfloat16
layer = torch.nn.Module()
layer.weight = Parameter(
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
layer.weight_scale = Parameter(
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
kernel.process_weights_after_loading(layer)
assert kernel.use_asm_gemm is expected
if not expected:
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
assert layer.weight_scale.is_contiguous()
@@ -1,286 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Native CDNA4 MXFP8 linear (``RocmDotScaledMxfp8LinearKernel``).
DeepSeek V4.1 routes every attention linear through this kernel. It used to
carry two BF16-dequantize workarounds on ROCm (one for all ``.attn.`` linears,
one for the row-parallel ``wo_b``, whose comment blamed NaNs at small token
counts). These tests pin the two properties those workarounds stood in for --
finite output and MXFP8-level accuracy -- across the V4.1 attention shapes at
TP=4 and the small token counts decode actually runs.
"""
import pytest
import torch
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
_mxfp8_e4m3_quantize_triton,
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm() or not current_platform.supports_mx(),
reason="native MX linear requires CDNA4 (gfx95x)",
)
# (name, N, K) for the DeepSeek V4.1 attention linears at tensor_parallel=4:
# fused wq_a|wkv is replicated, wq_b/wo_a are column-parallel, wo_b is
# row-parallel (so K, not N, is sharded).
V41_ATTN_SHAPES = [
("fused_wqa_wkv", 1280 + 512, 5120),
("wq_b", 64 * 512 // 4, 1280),
("wo_a", 8 * 1024 // 4, 4096),
("wo_b", 5120, 8192 // 4),
]
def _make_layer(n: int, k: int, device: str) -> torch.nn.Module:
weight_bf16 = torch.randn(n, k, device=device, dtype=torch.bfloat16) * 0.05
weight, weight_scale = _mxfp8_e4m3_quantize_torch(weight_bf16)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
return layer
@pytest.mark.parametrize("name,n,k", V41_ATTN_SHAPES, ids=lambda v: str(v))
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 16, 31, 32, 128, 1024])
@torch.inference_mode()
def test_rocm_mxfp8_linear_matches_dequant_reference(name, n, k, num_tokens):
"""Output is finite and matches an exact MXFP8 GEMM of the same inputs.
The reference quantizes the activation with the same quantizer and then
contracts in FP32, so the tolerance covers only kernel/accumulation error,
not the MXFP8 quantization error both sides share.
"""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
torch.manual_seed(num_tokens)
device = "cuda"
layer = _make_layer(n, k, device)
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
kernel.process_weights_after_loading(layer)
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
out = kernel.apply_weights(layer, x)
assert out.shape == (num_tokens, n)
assert torch.isfinite(out).all(), f"{name}: non-finite output"
x_q, x_scale = mxfp8_e4m3_quantize(x)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
)
rel = (out.float() - expected).norm() / expected.norm()
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
@torch.inference_mode()
def test_rocm_mxfp8_linear_handles_extreme_magnitudes():
"""Rows that are all-zero or near the E4M3 range must not produce NaNs.
A degenerate block makes the dynamic activation quantizer pick an extreme
E8M0 scale; a zero row makes it pick the tiny-clamped one. Both feed
``tl.dot_scaled`` directly.
"""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
torch.manual_seed(0)
device = "cuda"
n, k = 5120, 2048 # wo_b at TP=4
layer = _make_layer(n, k, device)
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
kernel.process_weights_after_loading(layer)
x = torch.randn(8, k, device=device, dtype=torch.bfloat16)
x[0] = 0.0
x[1] = 1e-30
x[2] = 400.0
x[3, ::2] = 0.0
out = kernel.apply_weights(layer, x)
assert torch.isfinite(out).all()
assert (out[0] == 0).all(), "an all-zero activation row must give a zero row"
# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The
# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has
# K % 128 != 0, which only the block-scaled GEMM runs natively.
V41_BLOCK32_SHAPES = [
("fused_wqa_wkv", 1280 + 512, 5120),
("wq_b_tp2", 64 * 512 // 2, 1280),
("indexer_wq_b", 32 * 128, 1280),
("wo_b_tp2", 5120, 8192 // 2),
("shared_gate_up_tp2", 2 * 2304 // 2, 5120),
("shared_down_tp2", 5120, 2304 // 2),
("shared_down_tp4", 5120, 2304 // 4),
]
def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module:
"""An MXFP8 layer with 32x32 block scales, expanded per row as loaded."""
blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32)
amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny)
bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254)
weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
weight.to(torch.float8_e4m3fn), requires_grad=False
)
layer.weight_scale = torch.nn.Parameter(
bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False
)
return layer
@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v))
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024])
@torch.inference_mode()
def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens):
"""32x32 block scales stay compact and the block-scaled GEMM is exact.
The token counts cover the packed, tiled and split-K configurations.
"""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
torch.manual_seed(num_tokens)
device = "cuda"
layer = _make_block32_layer(n, k, device)
row_scale = layer.weight_scale.data.clone()
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
kernel.process_weights_after_loading(layer)
assert layer.weight.dtype == torch.float8_e4m3fn
assert layer.weight_scale.shape == (n // 32, k // 32)
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
out = kernel.apply_weights(layer, x)
x_q, x_scale = mxfp8_e4m3_quantize(x)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T
)
assert out.shape == (num_tokens, n)
rel = (out.float() - expected).norm() / expected.norm()
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
@torch.inference_mode()
def test_rocm_mxfp8_linear_keeps_per_row_scales():
"""Scales that differ within a 32-row block are not compacted."""
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
n, k = 64, 256
layer = _make_block32_layer(n, k, "cuda")
layer.weight_scale.data[1, 0] += 1
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
kernel.process_weights_after_loading(layer)
assert layer.weight_scale.shape == (n, k // 32)
def _split_k_case(packed: bool):
"""A 3-way in-launch split-K on shared_down TP4, with K and M tails."""
from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g
n, k, m = 5120, 576, 19
layer = _make_block32_layer(n, k, "cuda")
x_q, x_scale = mxfp8_e4m3_quantize(
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
)
expected = (
dequant_mxfp8_to_bf16(x_q, x_scale).float()
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
)
if packed:
cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3)
else:
cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3)
args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous())
return g, cfg, args, expected
def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float:
return ((out.float() - expected).norm() / expected.norm()).item()
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_in_launch(packed):
"""Splits summed by the tile's last program are exact and repeatable."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.empty_like(expected, dtype=torch.bfloat16)
g._launch(*args, out, cfg)
assert _rel_err(out, expected) < 5e-3
first = out.clone()
for _ in range(20):
g._launch(*args, out, cfg)
assert torch.equal(out, first)
@pytest.mark.parametrize("packed", [True, False])
@torch.inference_mode()
def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed):
"""A stream whose first split-K launch is being captured has no counters
yet; it must fall back to an unfused split instead of failing."""
torch.manual_seed(0)
g, cfg, args, expected = _split_k_case(packed)
out = torch.zeros_like(expected, dtype=torch.bfloat16)
stream = torch.cuda.Stream()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
g._launch(*args, out, cfg)
graph.replay()
torch.accelerator.synchronize()
assert _rel_err(out, expected) < 5e-3
@pytest.mark.parametrize("num_rows", [1, 64, 65])
@torch.inference_mode()
def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows):
"""The fused ROCm quantizer must agree with the torch one, zeros included.
An all-zero 32-element block clamps the E8M0 scale to 0, i.e. 2**-127 --
subnormal in fp32 and flushed to zero on CDNA. Dividing by it turned the
block into 0/0 == NaN, which then poisoned every output column of the GEMM
that consumed it.
"""
torch.manual_seed(num_rows)
device = "cuda"
x = torch.randn(num_rows, 128, device=device, dtype=torch.bfloat16)
x[0] = 0.0 # all-zero row: every block clamps to scale byte 0
x[-1, :32] = 0.0 # single zero block in an otherwise normal row
x[-1, 32:64] = 1e-38 # denormal-ish block
q_triton, s_triton = _mxfp8_e4m3_quantize_triton(x)
q_torch, s_torch = _mxfp8_e4m3_quantize_torch(x)
assert not torch.isnan(q_triton.float()).any()
torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0)
torch.testing.assert_close(
q_triton.view(torch.uint8), q_torch.view(torch.uint8), rtol=0, atol=0
)
@@ -1,416 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import ref_dynamic_per_tensor_fp8_quant
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
from vllm.utils.platform_utils import num_compute_units
# Global per-test cleanup costs more than the tests themselves.
# These tests can just cleanup once at the end
pytestmark = pytest.mark.skip_global_cleanup
SEEDS = [0]
# These options are independent in the kernels, so the sets below cover every
# pair of them instead of the full product. bias_mode: 0 = none, 1 = (m,),
# 2 = (n, m).
OPTIONS_WVSPLITKRC = [
# dtype, padded_a, bias_mode, xnorm
(torch.float16, False, 0, False),
(torch.float16, True, 1, True),
(torch.float16, True, 2, False),
(torch.bfloat16, True, 0, True),
(torch.bfloat16, False, 1, False),
(torch.bfloat16, False, 2, True),
]
OPTIONS_WVSPLITK = [
# dtype, padded_a, padded_b, bias_mode, xnorm
(torch.float16, False, False, 0, False),
(torch.float16, False, True, 1, True),
(torch.float16, True, False, 2, True),
(torch.bfloat16, True, True, 0, True),
(torch.bfloat16, True, False, 1, False),
(torch.bfloat16, False, True, 2, False),
]
OPTIONS_WVSPLITK_FP8 = [
# dtype, padded_a, padded_b, biased, xnorm
(torch.float16, False, False, True, True),
(torch.bfloat16, True, False, False, True),
(torch.bfloat16, False, True, True, True),
(torch.float16, True, False, True, False),
(torch.float16, False, True, False, False),
(torch.bfloat16, True, True, False, False),
]
DTYPES = [torch.bfloat16, torch.float16]
# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow
# workspaces.
WVSPLITKRC_SLOTS = 8
# The CU count is the grid size. Production passes num_compute_units(); a small
# value manufactures better concurrency for these tests.
WVSPLITKRC_TEST_CU = 16
# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates.
_GATE_CYCLES = 300_000_000
# Specific (N, K, M) combinations for targeted testing
NKM_FACTORS_LLMM1 = [
# Small, medium, large cases
(1, 8, 16),
(1, 32, 64),
(1, 128, 256),
(1, 512, 1024),
(1, 2048, 4096),
# Edge cases with specific K sizes
(1, 6144, 1024),
(1, 8192, 2048),
# Very large case
(1, 4096, 8192),
]
NKM_FACTORS_WVSPLITK = [
# Different batch sizes with key dimensions
(1, 32, 16),
(1, 64, 64),
(2, 256, 256),
(3, 1024, 1024),
(4, 4096, 4096),
(4, 4096, 4096 + 1),
(4, 4096 + 16, 4096),
(4, 4096 + 16, 4096 + 1),
# Extended K values
(1, 9216, 512),
(2, 10240, 1024),
(4, 16384, 8192),
(4, 16384 * 2, 8192),
(4, 16384 * 2, 8192 + 1),
(4, 16384 * 2 + 16, 8192),
(4, 16384 * 2 + 16, 8192 + 1),
# Minimum M constraint validation (m >= 8)
(1, 64, 8),
(2, 128, 8),
(4, 256, 8),
]
# N is bucketed up to its next ^2 (16/32/64/128) and the remainder is masked
N_FACTORS_WVSPLITKRC = [
13,
16,
17,
25,
29,
31,
32,
41,
51,
64,
71,
81,
91,
103,
117,
128,
]
# K shards are 512 wide, evenly divided or not, +8 for a partial 8-element load
K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8]
# M tiles are 64 rows, +16 for a partial tile
M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
# (N, K, M) with more K-shards than the readback can stage in one LDS pass.
# Spans both CHUNKK values and both N-tile counts.
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
(128, 6144, 128),
(96, 8192, 128),
# Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs.
(128, 12288, 64),
(32, 12288, 128),
]
NKM_FACTORS_WVSPLITK_FP8 = [
# FP8-specific cases with K % 16 == 0
(1, 16, 16),
(1, 32, 16 + 16),
(1, 64, 64),
(1, 64, 64 + 16),
(1, 64 + 16, 64),
(1, 64 + 16, 64 + 16),
(4, 64, 64),
(4, 64, 64 + 16),
(4, 64 + 16, 64),
(4, 64 + 16, 64 + 16),
(2, 512, 512),
(3, 512, 512),
(3, 512, 512 + 16),
(4, 512, 512),
(3, 2048, 2048),
(3, 2048, 2048 + 16),
(4, 2048 + 16, 2048),
(4, 2048 + 16, 2048 + 16),
(4, 4096, 4096),
(4, 16400, 2048),
(4, 16400, 2048 + 16),
# Extended FP8 dimensions not covered by WVSPLITK
(1, 14336, 1024),
(2, 24576, 2048),
(4, 32768, 28672),
(4, 32768 * 2, 28672),
(4, 32768 * 2, 28672 + 16),
(4, 32768 * 2 + 16, 28672),
(4, 32768 * 2 + 16, 28672 + 16),
]
@pytest.fixture(scope="module", autouse=True)
def cleanup_after_all_tests():
yield
cleanup_dist_env_and_memory()
def pad_fp8(weight):
num_pad = 256 // weight.element_size()
import torch.nn.functional as F
return F.pad(weight, (0, num_pad), "constant", 0)[..., :-num_pad]
def make_bias(bias_mode, n, m, dtype):
if bias_mode == 0:
return None
shape = (m,) if bias_mode == 1 else (n, m)
return torch.rand(shape, dtype=dtype, device="cuda") * 2 - 1
@pytest.mark.parametrize("n", N_FACTORS_WVSPLITKRC)
@pytest.mark.parametrize("k", K_FACTORS_WVSPLITKRC)
@pytest.mark.parametrize("m", M_FACTORS_WVSPLITKRC)
@pytest.mark.parametrize("dtype,padded_a,bias_mode,xnorm", OPTIONS_WVSPLITKRC)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed):
torch.manual_seed(seed)
cu_count = num_compute_units()
# Next ^2 of n
N_p2 = 1 << (n - 1).bit_length()
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
# and each working on a 512-shard of K, how many CUs would we need?
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
# How many of 4 waves in a group can work on same 16 Ms at same time?
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
GrpsShrB = min(N_p2 // 16, 4)
# Given the above, how many CUs would we need?
CuNeeded = rndup_cus * GrpsShrB
# Deterministic reduction stores one float workspace value per K shard.
fits_wvsplitkrc = (N_p2 * m * ((k + 512 - 1) // 512)) <= 128 * 1024 * 12
fits_wvsplitkrc &= CuNeeded <= cu_count
if not fits_wvsplitkrc:
pytest.skip("Too large for wvSplitKrc")
xavier = (
math.sqrt(2 / k) if xnorm else 1
) # normalize to avoid large output-bias deltas
A = torch.randn(n, k, dtype=dtype, device="cuda") * xavier
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
if padded_a:
A = pad_fp8(A)
BIAS = make_bias(bias_mode, n, m, dtype)
ref_out = torch.nn.functional.linear(A, B, BIAS)
out = ops.wvSplitKrc(A, B, cu_count, BIAS)
if xnorm:
# The O(1) bias lifts outputs to ~O(1), where one bf16 ULP (~3.9e-3, the
# worst measured divergence) exceeds 1e-3. Bump atol to 5e-3 only for
# biased bf16 (above that ULP, still under finfo(bf16).eps); keep 1e-3
# otherwise.
atol = 5e-3 if (dtype == torch.bfloat16 and BIAS is not None) else 1e-3
torch.testing.assert_close(out, ref_out, atol=atol, rtol=1e-8)
else:
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed):
"""K large enough that the split-K readback must stage LDS in batches."""
torch.manual_seed(seed)
xavier = math.sqrt(2 / k)
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
ref_out = torch.nn.functional.linear(A, B, None)
out = ops.wvSplitKrc(A, B, num_compute_units(), None)
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4])
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed):
"""Concurrent streams must not share a split-K workspace.
The workspace is cleared by protocol rather than per invocation, so sharing
one corrupts the reduction and returns wrong numbers rather than failing.
n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path.
"""
torch.manual_seed(seed)
n, k, m = 16, 2048, 256
iters = 4
xavier = math.sqrt(2 / k)
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
As = [
torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams)
]
refs = [torch.nn.functional.linear(A, B, None) for A in As]
torch.accelerator.synchronize()
streams = [torch.Stream() for _ in range(n_streams)]
outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)]
_release_together(streams)
# Round-robin the enqueues so kernels from different streams are in flight
# at the same time.
for _ in range(iters):
for i, s in enumerate(streams):
with torch.cuda.stream(s):
outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None))
for s in streams:
torch.accelerator.current_stream().wait_stream(s)
torch.accelerator.synchronize()
for i in range(n_streams):
for out in outs[i]:
torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2)
def _release_together(streams):
"""Release every stream at once, once everything below is enqueued.
An event recorded on an idle stream is already complete, so waiting on it
gates nothing -- the spin, not the event, is what holds the streams.
"""
blocker = torch.Stream()
with torch.cuda.stream(blocker):
torch.cuda._sleep(_GATE_CYCLES)
gate = torch.Event()
gate.record()
for s in streams:
s.wait_event(gate)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16])
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
@torch.inference_mode()
def test_rocm_llmm1_kernel(n, k, m, dtype, rows_per_block, seed):
torch.manual_seed(seed)
# TODO: Zero-centering the inputs causes errors for LLMM1!
# Without that the numbers quickly saturate, and may
# be giving false matches.
A = torch.rand(n, k, dtype=dtype, device="cuda")
B = torch.rand(m, k, dtype=dtype, device="cuda")
ref_out = torch.matmul(A, B.t())
out = ops.LLMM1(B, A, rows_per_block)
torch.testing.assert_close(out, ref_out, atol=1e-8, rtol=1e-2)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITK)
@pytest.mark.parametrize(
"dtype,padded_a,padded_b,bias_mode,xnorm",
OPTIONS_WVSPLITK,
)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
def test_rocm_wvsplitk_kernel(
n, k, m, dtype, padded_a, padded_b, bias_mode, xnorm, seed
):
torch.manual_seed(seed)
cu_count = num_compute_units()
xavier = (
math.sqrt(2 / k) if xnorm else 1
) # normalize to avoid large output-bias deltas
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
BIAS = make_bias(bias_mode, n, m, dtype)
if padded_a:
A = pad_fp8(A)
if padded_b:
B = pad_fp8(B)
ref_out = torch.nn.functional.linear(A, B, BIAS)
out = ops.wvSplitK(B, A.view(-1, A.size(-1)), cu_count, BIAS)
# Accumulation error in fp16 GEMM scales with sqrt(K)
atol = torch.finfo(dtype).eps * math.sqrt(k)
torch.testing.assert_close(out, ref_out, atol=atol, rtol=1e-2)
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITK_FP8)
@pytest.mark.parametrize(
"dtype,padded_a,padded_b,biased,xnorm",
OPTIONS_WVSPLITK_FP8,
)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.skipif(
not (current_platform.is_rocm() and current_platform.supports_fp8()),
reason="only test for rocm fp8",
)
def test_rocm_wvsplitk_fp8_kernel(
n, k, m, dtype, padded_a, padded_b, biased, xnorm, seed
):
torch.manual_seed(seed)
xavier = math.sqrt(2 / k) if xnorm else 1 # normalize to avoid large deltas
A = (torch.rand(n, k, device="cuda") * 2 - 1) * xavier
B = (torch.rand(m, k, device="cuda") * 2 - 1) * xavier
A, scale_a = ref_dynamic_per_tensor_fp8_quant(A)
B, scale_b = ref_dynamic_per_tensor_fp8_quant(B)
if padded_b:
B = pad_fp8(B)
if padded_a:
A = pad_fp8(A)
BIAS = None if (not biased) else (torch.rand(m, dtype=dtype, device="cuda") * 2 - 1)
ref_out = torch._scaled_mm(
A, B.t(), out_dtype=dtype, scale_a=scale_a, scale_b=scale_b, bias=BIAS
)
out = ops.wvSplitKQ(B, A, dtype, scale_a, scale_b, num_compute_units(), BIAS)
if xnorm:
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-8)
elif k >= 32 * 1024:
# wider pytrch thresh for large-K & no xnorm
torch.testing.assert_close(out, ref_out, atol=0.07, rtol=5e-2)
else:
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ScaledMM kernel selection logic (CPU-only).
Run `pytest tests/kernels/quantization/test_scaled_mm_kernel_selection.py`.
"""
import inspect
from abc import ABC
from unittest.mock import patch
import pytest
import torch
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.kernels.linear import (
_POSSIBLE_KERNELS,
_POSSIBLE_NVFP4_KERNELS,
AiterInt8ScaledMMLinearKernel,
CPUInt8ScaledMMLinearKernel,
HummingFP8ScaledMMLinearKernel,
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
ScaledMMLinearKernel,
_get_linear_backend,
_resolve_backend_kernels,
init_fp8_linear_kernel,
init_int8_linear_kernel,
register_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import PlatformEnum
pytestmark = pytest.mark.cpu_test
def test_auto_linear_backend_is_unchanged_without_overrides():
kernels: list[type] = [object]
quantizations = (
"fp8_block_w8a8",
"fp8_w8a8",
"int8_w8a8",
"mixed_precision",
"mxfp8",
"mxfp4",
"mxfp6",
"w8a16_fp8",
"nvfp4_w4a4",
"nvfp4_w4a16",
)
config = VllmConfig(kernel_config=KernelConfig(linear_backend="auto"))
with set_current_vllm_config(config):
for quantization in quantizations:
assert _get_linear_backend(quantization=quantization) == "auto"
assert (
_resolve_backend_kernels(kernels, "test", quantization=quantization)
is kernels
)
def test_linear_backend_override_is_quantization_specific():
config = VllmConfig(
kernel_config=KernelConfig(
linear_backend="cutlass",
linear_backend_per_quant={"nvfp4_w4a16": "humming"},
)
)
with set_current_vllm_config(config):
assert _get_linear_backend(quantization="nvfp4_w4a16") == "humming"
assert _get_linear_backend(quantization="nvfp4_w4a4") == "cutlass"
assert _get_linear_backend(quantization="fp8_w8a8") == "cutlass"
@patch.object(HummingFP8ScaledMMLinearKernel, "is_supported", return_value=(True, None))
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_fp8_linear_backend_override(platform_mock, _):
platform_mock._enum = PlatformEnum.CUDA
config = VllmConfig(
kernel_config=KernelConfig(
linear_backend="cutlass",
linear_backend_per_quant={"fp8_w8a8": "humming"},
)
)
with set_current_vllm_config(config):
kernel = init_fp8_linear_kernel(
activation_quant_key=kFp8StaticTensorSym,
weight_quant_key=kFp8StaticTensorSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
weight_shape=(128, 128),
)
assert type(kernel) is HummingFP8ScaledMMLinearKernel
def test_is_supported_is_abstract():
"""Test that is_supported() is properly defined as abstract."""
assert issubclass(ScaledMMLinearKernel, ABC)
assert hasattr(ScaledMMLinearKernel, "is_supported")
def test_cpu_kernel_implements_is_supported():
"""Test that CPUInt8ScaledMMLinearKernel implements is_supported() method."""
assert hasattr(CPUInt8ScaledMMLinearKernel, "is_supported"), (
"CPUInt8ScaledMMLinearKernel missing is_supported() method"
)
# Verify it's a classmethod by checking if it can be called with the class
# and by checking the method type
assert inspect.ismethod(
CPUInt8ScaledMMLinearKernel.is_supported
) or inspect.isfunction(CPUInt8ScaledMMLinearKernel.is_supported), (
"CPUInt8ScaledMMLinearKernel.is_supported() should be a classmethod"
)
# Verify it can be called as a classmethod
result, reason = CPUInt8ScaledMMLinearKernel.is_supported()
assert isinstance(result, bool), "is_supported() should return a bool"
assert reason is None or isinstance(reason, str), "reason should be str or None"
def test_aiter_kernel_implements_is_supported():
"""Test that AiterInt8ScaledMMLinearKernel implements is_supported() method."""
assert hasattr(AiterInt8ScaledMMLinearKernel, "is_supported"), (
"AiterInt8ScaledMMLinearKernel missing is_supported() method"
)
# Verify it's a classmethod by checking if it can be called with the class
# and by checking the method type
assert inspect.ismethod(
AiterInt8ScaledMMLinearKernel.is_supported
) or inspect.isfunction(AiterInt8ScaledMMLinearKernel.is_supported), (
"AiterInt8ScaledMMLinearKernel.is_supported() should be a classmethod"
)
# Verify it can be called as a classmethod
# (will return False on CPU, which is expected)
result, reason = AiterInt8ScaledMMLinearKernel.is_supported()
assert isinstance(result, bool), "is_supported() should return a bool"
assert reason is None or isinstance(reason, str), "reason should be str or None"
# On CPU, it should return False with a reason about requiring ROCm
# This validates the method works correctly even on non-ROCm platforms
def test_cpu_kernel_accepts_all_configs():
"""Test that CPUInt8ScaledMMLinearKernel accepts all config combinations."""
configs = [
Int8ScaledMMLinearLayerConfig(
is_channelwise=False,
is_static_input_scheme=True,
input_symmetric=True,
),
Int8ScaledMMLinearLayerConfig(
is_channelwise=True,
is_static_input_scheme=False,
input_symmetric=False,
),
]
for config in configs:
can_impl, reason = CPUInt8ScaledMMLinearKernel.can_implement(config)
assert can_impl, (
f"CPUInt8ScaledMMLinearKernel should accept config {config}: {reason}"
)
class OOTInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
return True, None
@classmethod
def can_implement(cls, c: Int8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
pass
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
pass
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_register_oot_linear_kernel(platform_mock):
"""Test that the linear kernel registration works correctly."""
platform_mock._enum = PlatformEnum.OOT
register_linear_kernel(OOTInt8ScaledMMLinearKernel, PlatformEnum.OOT, "int8")
kernel = init_int8_linear_kernel(True, True, True, "module")
assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), (
"init_int8_linear_kernel should return an instance of the registered kernel"
)
@pytest.mark.parametrize(
"possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS]
)
def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels):
"""Preferring Humming must not demote Marlin below unrelated kernels."""
kernels = possible_kernels[PlatformEnum.CUDA]
original = list(kernels)
assert prioritize_humming(kernels, compute_capability=80) == original
names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)]
humming = next(i for i, name in enumerate(names) if "Humming" in name)
assert "Marlin" in names[humming + 1]
del names[humming]
assert names == [k.__name__ for k in original if "Humming" not in k.__name__]
assert kernels == original
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
dequantize_nvfp4_to_dtype,
)
from vllm._custom_ops import scaled_fp4_quant
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
if not current_platform.has_device_capability(100):
pytest.skip(
reason="Nvfp4 Requires compute capability of 10 or above.",
allow_module_level=True,
)
FP4_DTYPE = torch.uint8
FP8_DTYPE = current_platform.fp8_dtype()
DTYPES = [torch.float16, torch.bfloat16]
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128), (1, 256), (64, 256)]
BLOCK_SIZE = 16
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("shape", SHAPES)
@torch.inference_mode()
def test_silu_mul_nvfp4_quant(
default_vllm_config,
dtype: torch.dtype,
shape: tuple[int, int],
) -> None:
set_random_seed(42)
device = "cuda:0"
torch.set_default_device(device)
x = torch.randn(shape, dtype=dtype)
# ref op
ref_output = SiluAndMul().forward_native(x)
ref_global_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.abs(
ref_output
).max().to(torch.float32)
ref_output_quant, ref_block_scale = scaled_fp4_quant(ref_output, ref_global_scale)
# fused op
fused_output_quant = torch.empty_like(ref_output_quant)
fused_block_scale = torch.empty_like(ref_block_scale)
torch.ops._C.silu_and_mul_nvfp4_quant(
fused_output_quant, fused_block_scale, x, ref_global_scale
)
# check dtype
assert ref_output_quant.dtype == FP4_DTYPE
assert fused_output_quant.dtype == FP4_DTYPE
assert ref_output_quant.shape == fused_output_quant.shape
assert ref_block_scale.dtype == FP8_DTYPE
assert fused_block_scale.dtype == FP8_DTYPE
assert ref_block_scale.shape == fused_block_scale.shape
# check dequantized output
ref_output_dequant = dequantize_nvfp4_to_dtype(
ref_output_quant, ref_block_scale, ref_global_scale, dtype, device
)
fused_output_dequant = dequantize_nvfp4_to_dtype(
fused_output_quant, fused_block_scale, ref_global_scale, dtype, device
)
atol, rtol = 3e-1, 3e-1
torch.testing.assert_close(
ref_output_dequant, fused_output_dequant, atol=atol, rtol=rtol
)
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the triton_scaled_mm kernel.
Run `pytest tests/kernels/quantization/test_triton_scaled_mm.py`.
"""
import importlib
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
device = current_platform.device_type
triton_scaled_mm_module = importlib.import_module(
"vllm.model_executor.layers.quantization.compressed_tensors.triton_scaled_mm"
)
triton_scaled_mm = triton_scaled_mm_module.triton_scaled_mm
def torch_scaled_mm(
a: torch.Tensor,
b: torch.Tensor,
scale_a: torch.Tensor,
scale_b: torch.Tensor,
out_dtype: type[torch.dtype],
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
out = scale_a * out
out = scale_b.T * out
if bias is not None:
out = out + bias.float()
return out.to(out_dtype)
def get_8bit_types():
types = [torch.int8]
if current_platform.supports_fp8():
types.append(current_platform.fp8_dtype())
return types
# This test is to check regressions for int8 support on ROCm.
@pytest.mark.parametrize(
"model_path",
[
"neuralmagic/Llama-3.2-1B-quantized.w8a8",
],
)
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [10])
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm")
def test_rocm_compressed_tensors_w8a8(
vllm_runner, example_prompts, model_path, max_tokens, num_logprobs
):
dtype = "bfloat16"
with vllm_runner(model_path, dtype=dtype) as vllm_model:
vllm_model.generate_greedy_logprobs(example_prompts, max_tokens, num_logprobs)
MNK_FACTORS = [
(1, 256, 128),
(33, 256, 496),
(64, 971, 1024),
(64, 20486, 128),
(512, 256, 496),
(512, 20486, 1024),
]
@pytest.mark.parametrize("M,N,K", MNK_FACTORS)
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_scalar_scale_a", [True, False])
@pytest.mark.parametrize("use_scalar_scale_b", [True, False])
@pytest.mark.parametrize("use_bias", [True, False])
def test_scaled_mm(
M, N, K, in_dtype, out_dtype, use_scalar_scale_a, use_scalar_scale_b, use_bias
):
is_floating_point_type = lambda t: torch.tensor([1, 1], dtype=t).is_floating_point()
set_random_seed(0)
# NOTE: There are cases, where if the matrix is large enough, an output
# like 65504.4 can be produced, and can easily turn into inf when
# multiplied when using float16/bfloat16. This means one function, e.g.,
# testing function, and another function, e.g. golden function, can
# produce a non-inf value while the other produces an inf value, and
# will cause assert_close/allclose to fail, even though if overflow
# wouldn't have occurred, the values would have been "close."
#
# So, the values here are kept small enough to avoid this situation.
if is_floating_point_type(in_dtype):
a = (0.25 * torch.rand((M, K), dtype=torch.float32, device=device)).to(in_dtype)
b = (0.25 * torch.rand((K, N), dtype=torch.float32, device=device)).to(in_dtype)
else:
a = torch.randint(-32, 32, (M, K), dtype=in_dtype, device=device)
b = torch.randint(-32, 32, (K, N), dtype=in_dtype, device=device)
if use_scalar_scale_a:
scale_a = torch.rand((1, 1), device=device)
else:
scale_a = 0.25 * torch.rand((M, 1), device=device)
if use_scalar_scale_b:
scale_b = torch.rand((1, 1), device=device)
else:
scale_b = 0.25 * torch.rand((N, 1), device=device)
bias = None
if use_bias:
bias = torch.rand((N,), device=device, dtype=out_dtype)
c_check = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
c_actual = torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_heuristic", [True, False])
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
a = torch.ones((1, 32), device=device).to(in_dtype)
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
scale_a = torch.ones((1, 1), device=device)
scale_b = torch.full((1, 1), 2**-10, device=device)
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
expected = torch_scaled_mm(*args)
assert (expected == 1 + 2**-7).all()
torch.testing.assert_close(out, expected, rtol=0, atol=0)
# TD operand loads must be bit-exact vs the plain masked-load path.
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Triton scaled_mm runs on CUDA-alike or XPU.",
)
@pytest.mark.parametrize(
"M,N,K", [(1, 4096, 4096), (64, 4096, 4096), (256, 2048, 4096)]
)
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_scalar_scale_a", [True, False])
@pytest.mark.parametrize("use_bias", [True, False])
def test_scaled_mm_td_matches_plain(M, N, K, in_dtype, use_scalar_scale_a, use_bias):
dev = current_platform.device_type
out_dtype = torch.bfloat16
set_random_seed(0)
is_fp = torch.tensor([1, 1], dtype=in_dtype).is_floating_point()
if is_fp:
a = (0.25 * torch.rand((M, K), dtype=torch.float32, device=dev)).to(in_dtype)
b = (0.25 * torch.rand((K, N), dtype=torch.float32, device=dev)).to(in_dtype)
else:
a = torch.randint(-32, 32, (M, K), dtype=in_dtype, device=dev)
b = torch.randint(-32, 32, (K, N), dtype=in_dtype, device=dev)
scale_a = (
torch.rand((1, 1), device=dev)
if use_scalar_scale_a
else 0.25 * torch.rand((M, 1), device=dev)
)
scale_b = 0.25 * torch.rand((N, 1), device=dev)
bias = torch.rand((N,), device=dev, dtype=out_dtype) if use_bias else None
out_plain = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias, use_td=False)
out_td = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias, use_td=True)
torch.testing.assert_close(out_td, out_plain, rtol=0, atol=0)
@@ -1,490 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the ROCm Triton W4A16 GEMM kernel.
Run `pytest tests/kernels/quantization/test_triton_w4a16.py`.
"""
import importlib
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
# This test module is ROCm/Triton specific. Avoid import-time failures on
# non-ROCm or environments without Triton by skipping early.
if not current_platform.is_rocm():
pytest.skip("ROCm only", allow_module_level=True)
pytest.importorskip("triton")
device = "cuda"
triton_w4a16_module = importlib.import_module(
"vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16"
)
triton_w4a16_gemm = triton_w4a16_module.triton_w4a16_gemm
TritonW4A16LinearKernel = triton_w4a16_module.TritonW4A16LinearKernel
def _pack_int4_along_n(w_int4_kn: torch.Tensor) -> torch.Tensor:
"""Pack int4 values along N: [K, N] -> [K, N//8] int32."""
assert w_int4_kn.dtype == torch.int32
K, N = w_int4_kn.shape
assert N % 8 == 0
shifts = torch.arange(8, device=w_int4_kn.device, dtype=torch.int32) * 4
return torch.sum(
(w_int4_kn.view(K, N // 8, 8) & 0xF) << shifts,
dim=2,
dtype=torch.int32,
).contiguous()
def _unpack_int4_along_n(w_packed_kn8: torch.Tensor) -> torch.Tensor:
"""Unpack int4 values along N: [K, N//8] -> [K, N] int32."""
assert w_packed_kn8.dtype == torch.int32
K, N8 = w_packed_kn8.shape
shifts = torch.arange(8, device=w_packed_kn8.device, dtype=torch.int32) * 4
nibbles = (w_packed_kn8.unsqueeze(-1) >> shifts) & 0xF
return nibbles.reshape(K, N8 * 8)
def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor:
"""Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8]."""
assert w_int4_kn.dtype == torch.int32
K, N = w_int4_kn.shape
assert K % 8 == 0
out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device)
for i in range(8):
out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4)
return out.contiguous()
def _w4a16_reference(
a_mk: torch.Tensor,
b_packed_kn8: torch.Tensor,
scales_gn: torch.Tensor,
*,
group_size: int,
qzeros_gn8: torch.Tensor | None,
zp_bias: int,
) -> torch.Tensor:
"""Reference implementation for W4A16.
a_mk: [M,K] fp16/bf16
b_packed_kn8: [K, N//8] int32, N-packed int4 weights
scales_gn: [K//G, N] fp16/bf16
qzeros_gn8: [K//G, N//8] int32, N-packed int4 zeros, or None
"""
assert a_mk.dtype in (torch.float16, torch.bfloat16)
assert b_packed_kn8.dtype == torch.int32
assert scales_gn.dtype == a_mk.dtype
M, K = a_mk.shape
N = b_packed_kn8.shape[1] * 8
assert b_packed_kn8.shape[0] == K
assert group_size > 0 and K % group_size == 0
G = group_size
num_groups = K // G
assert scales_gn.shape == (num_groups, N)
w_int4 = _unpack_int4_along_n(b_packed_kn8) # [K,N]
if qzeros_gn8 is None:
z_full = torch.full((K, N), zp_bias, dtype=torch.int32, device=a_mk.device)
else:
assert qzeros_gn8.shape == (num_groups, N // 8)
z_gn = _unpack_int4_along_n(qzeros_gn8) # [G,N] in groups
z_full = z_gn.repeat_interleave(G, dim=0) # [K,N]
s_full = scales_gn.repeat_interleave(G, dim=0).to(torch.float32) # [K,N]
w_fp = (w_int4 - z_full).to(torch.float32) * s_full # [K,N]
out = a_mk.to(torch.float32) @ w_fp # [M,N]
return out.to(a_mk.dtype)
def test_triton_w4a16_gemm_fake_impl():
with FakeTensorMode():
a = torch.empty((4, 256), device=device, dtype=torch.float16)
b_q = torch.empty((256, 64), device=device, dtype=torch.int32)
scales = torch.empty((2, 512), device=device, dtype=torch.float16)
out = triton_w4a16_gemm(
a=a,
b_q=b_q,
scales=scales,
qzeros=None,
group_size=128,
)
assert out.shape == (4, 512)
assert out.dtype == a.dtype
assert out.device == a.device
graph = make_fx(
lambda a, b_q, scales: triton_w4a16_gemm(
a, b_q, scales, qzeros=None, group_size=128
)
)(a, b_q, scales)
assert any(
node.op == "call_function"
and node.target == torch.ops.vllm.triton_w4a16_gemm.default
for node in graph.graph.nodes
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"M,K,N,G,has_zp",
[
(1, 256, 256, 32, False),
(17, 256, 512, 32, False),
(32, 512, 256, 64, False),
(33, 512, 512, 128, False),
(64, 1024, 256, 256, False),
(128, 256, 1024, 32, True),
(64, 512, 512, 64, True),
],
)
def test_triton_w4a16_gemm_matches_reference(dtype, M, K, N, G, has_zp):
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
if N % 8 != 0 or K % G != 0:
pytest.skip("Invalid test shape")
set_random_seed(0)
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
b_packed = _pack_int4_along_n(w_int4)
scales = (0.05 * torch.rand((K // G, N), device=device, dtype=torch.float32)).to(
dtype
)
qzeros = None
if has_zp:
zeros_int4 = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
qzeros = _pack_int4_along_n(zeros_int4)
out = triton_w4a16_gemm(
a=a,
b_q=b_packed,
scales=scales,
qzeros=qzeros,
group_size=G,
zp_bias=8,
)
ref = _w4a16_reference(
a,
b_packed,
scales,
group_size=G,
qzeros_gn8=qzeros,
zp_bias=8,
)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_triton_w4a16_gemm_requires_contiguous_inputs():
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
set_random_seed(0)
M, K, N, G = 32, 256, 256, 32
a = torch.randn((K, M), device=device, dtype=torch.float16).t() # non-contiguous
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
b_packed = _pack_int4_along_n(w_int4)
scales = torch.rand((K // G, N), device=device, dtype=torch.float16)
with pytest.raises(AssertionError):
triton_w4a16_gemm(
a=a,
b_q=b_packed,
scales=scales,
qzeros=None,
group_size=G,
zp_bias=8,
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_triton_w4a16_process_weights_after_loading_repacks_layout():
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedColumnParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
with set_current_vllm_config(VllmConfig()):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method="tcp://127.0.0.1:0",
local_rank=0,
)
ensure_model_parallel_initialized(1, 1)
set_random_seed(0)
# Small-but-nontrivial shapes.
K, N = 256, 256
G = 32
assert K % 8 == 0 and N % 8 == 0 and K % G == 0
# Build a canonical int4 weight grid then pack into the CT checkpoint layout.
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn) # [N, K//8]
# Scales in CT checkpoint layout for WNA16: [N, K//G]
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
# Asymmetric case: zero points in CT checkpoint layout [N//8, K//G] (N-packed)
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
zeros_packed_gn8 = _pack_int4_along_n(zeros_int4_gn) # [K//G, N//8]
zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G]
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4, # asymmetric
act_type=torch.float16,
group_size=G,
zero_points=True,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="weight_packed",
w_s_param_name="weight_scale",
w_zp_param_name="weight_zero_point",
)
# Build dummy layer with vLLM parameter wrappers.
weight_loader = lambda *args, **kwargs: None
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"weight_packed",
PackedvLLMParameter(
data=w_ckpt_nk8,
weight_loader=weight_loader,
input_dim=1,
output_dim=0,
packed_factor=8,
packed_dim=1,
),
)
layer.register_parameter(
"weight_scale",
GroupQuantScaleParameter(
data=scales_ckpt_nkg,
weight_loader=weight_loader,
input_dim=1,
output_dim=0,
),
)
layer.register_parameter(
"weight_zero_point",
PackedColumnParameter(
data=zeros_ckpt_n8kg,
weight_loader=weight_loader,
output_dim=0,
packed_factor=8,
packed_dim=0,
),
)
kernel.process_weights_after_loading(layer)
# Expected transformed layouts.
expected_w_kn8 = _pack_int4_along_n(w_int4_kn) # [K, N//8]
expected_scales_gn = scales_ckpt_nkg.t().contiguous() # [K//G, N]
expected_zeros_gn8 = zeros_ckpt_n8kg.t().contiguous() # [K//G, N//8]
assert tuple(layer.weight_packed.shape) == (K, N // 8)
assert tuple(layer.weight_scale.shape) == (K // G, N)
assert tuple(layer.weight_zero_point.shape) == (K // G, N // 8)
torch.testing.assert_close(layer.weight_packed, expected_w_kn8)
torch.testing.assert_close(layer.weight_scale, expected_scales_gn)
torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
"""AutoGPTQ qzeros are already [K//G, N//8] (output_dim=1): no transpose."""
if not torch.cuda.is_available():
pytest.skip("CUDA/HIP device not available")
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
)
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
PackedvLLMParameter,
)
from vllm.scalar_type import scalar_types
with set_current_vllm_config(VllmConfig()):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method="tcp://127.0.0.1:0",
local_rank=0,
)
ensure_model_parallel_initialized(1, 1)
set_random_seed(0)
K, N = 256, 256
G = 32
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
w_gptq_k8n = _pack_int4_along_k_to_ckpt(w_int4_kn).t().contiguous()
scales_gptq_gn = 0.05 * torch.rand((K // G, N), device=device, dtype=torch.float16)
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
zeros_gptq_gn8 = _pack_int4_along_n(zeros_int4_gn)
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4,
act_type=torch.float16,
group_size=G,
zero_points=True,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
)
weight_loader = lambda *args, **kwargs: None
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.register_parameter(
"qweight",
PackedvLLMParameter(
data=w_gptq_k8n,
weight_loader=weight_loader,
input_dim=0,
output_dim=1,
packed_factor=8,
packed_dim=0,
),
)
layer.register_parameter(
"scales",
GroupQuantScaleParameter(
data=scales_gptq_gn,
weight_loader=weight_loader,
input_dim=0,
output_dim=1,
),
)
layer.register_parameter(
"qzeros",
PackedvLLMParameter(
data=zeros_gptq_gn8,
weight_loader=weight_loader,
input_dim=0,
output_dim=1,
packed_factor=8,
packed_dim=1,
),
)
kernel.process_weights_after_loading(layer)
expected_w_kn8 = _pack_int4_along_n(w_int4_kn)
assert tuple(layer.qweight.shape) == (K, N // 8)
assert tuple(layer.scales.shape) == (K // G, N)
assert tuple(layer.qzeros.shape) == (K // G, N // 8)
torch.testing.assert_close(layer.qweight, expected_w_kn8)
torch.testing.assert_close(layer.scales, scales_gptq_gn)
torch.testing.assert_close(layer.qzeros, zeros_gptq_gn8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
"""For symmetric (uint4b8) layers, apply_weights must pass qzeros=None."""
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.scalar_type import scalar_types
K, N, G = 256, 256, 32
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8,
act_type=torch.float16,
group_size=G,
zero_points=False,
)
kernel = TritonW4A16LinearKernel(
config,
w_q_param_name="qweight",
w_s_param_name="scales",
w_zp_param_name="qzeros",
)
class DummyLayer(torch.nn.Module):
pass
layer = DummyLayer()
layer.qweight = torch.empty((K, N // 8), device=device, dtype=torch.int32)
layer.scales = torch.empty((K // G, N), device=device, dtype=torch.float16)
layer.qzeros = torch.empty((1, 1), device=device, dtype=torch.int32)
captured = {}
def fake_gemm(*, a, b_q, scales, qzeros, group_size, zp_bias):
captured["qzeros"] = qzeros
captured["zp_bias"] = zp_bias
return torch.empty((a.shape[0], N), device=a.device, dtype=a.dtype)
monkeypatch.setattr(triton_w4a16_module, "triton_w4a16_gemm", fake_gemm)
x = torch.empty((2, K), device=device, dtype=torch.float16)
output = kernel.apply_weights(layer, x)
assert output.shape == (2, N)
assert captured["qzeros"] is None
assert captured["zp_bias"] == scalar_types.uint4b8.bias
@@ -1,70 +0,0 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for W4A16 kernel selection logic (ROCm).
Run `pytest tests/kernels/quantization/test_w4a16_kernel_selection.py`.
"""
import pytest
import torch
from vllm.model_executor.kernels.linear import (
MPLinearLayerConfig,
choose_mp_linear_kernel,
)
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx1x, on_gfx1100
else:
on_gfx1100 = on_gfx1x = lambda: False # noqa: E731
# Group sizes the HIP skinny path of RDNAHybridW4A16LinearKernel instantiates.
_HYBRID_GROUP_SIZES = (32, 64, 128)
def _expected_rocm_kernel(weight_type, group_size: int) -> str:
"""Mirror the ROCm priority order in ``_POSSIBLE_KERNELS[ROCM]``:
RDNA3 (gfx1100, symmetric uint4b8) -> Hybrid (gfx11/gfx12) -> Triton.
"""
if on_gfx1100() and weight_type == scalar_types.uint4b8:
return "RDNA3W4A16LinearKernel"
if on_gfx1x() and group_size in _HYBRID_GROUP_SIZES:
return "RDNAHybridW4A16LinearKernel"
return "TritonW4A16LinearKernel"
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_choose_mp_linear_kernel_uint4b8():
# int4 weights, 16-bit activations (CT W4A16 typical config).
K, N = 1024, 256
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4b8, # symmetric int4 (bias=8)
act_type=torch.float16,
group_size=128,
zero_points=False,
)
kernel_type = choose_mp_linear_kernel(config)
assert kernel_type.__name__ == _expected_rocm_kernel(scalar_types.uint4b8, 128)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_choose_mp_linear_kernel_uint4_asymmetric():
# Asymmetric int4 weights (explicit zero points).
K, N = 512, 512
config = MPLinearLayerConfig(
full_weight_shape=(K, N),
partition_weight_shape=(K, N),
weight_type=scalar_types.uint4, # asymmetric int4 (explicit zeros)
act_type=torch.bfloat16,
group_size=64,
zero_points=True,
)
kernel_type = choose_mp_linear_kernel(config)
assert kernel_type.__name__ == _expected_rocm_kernel(scalar_types.uint4, 64)
@@ -1,661 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
The zentorch ops are mocked with reference implementations when zentorch is not
installed, so the layout and dispatch contracts are covered in CI.
"""
import dataclasses
import pytest
import torch
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
_import_unpack_from_int32,
)
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
GROUP_SIZE = 128
IN_FEATURES = 512
OUT_FEATURES = 128
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
words = packed.view(torch.int32)
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
for i in range(8):
nibble = (words >> (4 * i)) & 0xF
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
return out
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
n, k = unpacked.shape
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
out = torch.zeros(n, k // 8, dtype=torch.int32)
for i in range(8):
out |= (values[:, :, i] & 0xF) << (4 * i)
return out
def _quantize_per_group(
weight: torch.Tensor, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
out_features, in_features = weight.shape
grouped = weight.float().reshape(
out_features, in_features // group_size, group_size
)
scale = grouped.abs().amax(dim=-1) / 8.0
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
def _dequantize_per_group(
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
) -> torch.Tensor:
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
def _dynamic_quant_matmul(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
out = (x_q.float() @ weight.t()) * scale
if bias is not None:
out = out + bias.float()
return out
@pytest.fixture
def mock_zentorch_ops():
"""Register reference zentorch ops when zentorch is not installed."""
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
yield None
return
calls: dict[str, tuple] = {}
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
in_features = weight.view(torch.int32).shape[1] * 8
group_size = in_features // weight_scales.shape[0]
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
weight_deq = _unpack_s4(weight, in_features) * scales
out = _dynamic_quant_matmul(input, weight_deq, bias)
return out.to(input.dtype)
def _fused_moe(
output,
input,
w13,
w2,
w13_bias,
w2_bias,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales=None,
w2_scales=None,
zentorch_op_name="",
):
calls["fused_moe"] = (
input,
w13,
w2,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales,
w2_scales,
)
output.zero_()
lib_def = torch.library.Library("zentorch", "DEF")
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
lib_def.define(
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
"Tensor weight_scales, Tensor? bias=None, *, "
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
)
lib_def.define(
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
"Tensor topk_id, bool skip_weighted, str act, "
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
)
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
lib_impl.impl("zentorch_fused_moe", _fused_moe)
yield calls
lib_impl._destroy()
lib_def._destroy()
def _make_config(
act_type: torch.dtype = torch.bfloat16,
weight_type=scalar_types.uint4b8,
zero_points: bool = False,
group_size: int = GROUP_SIZE,
) -> MPLinearLayerConfig:
return MPLinearLayerConfig(
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
weight_type=weight_type,
act_type=act_type,
group_size=group_size,
zero_points=zero_points,
)
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
torch.manual_seed(0)
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
quantized, scale = _quantize_per_group(weight, group_size)
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
packed = pack_to_int32(quantized, 4, packed_dim=1)
layer = torch.nn.Module()
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
layer.weight_packed.input_dim = 1
layer.weight_packed.packed_dim = 1
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
layer.weight_zero_point = None
return layer, _dequantize_per_group(quantized, scale, group_size)
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
return ZentorchWNA16LinearKernel(
config, "weight_packed", "weight_scale", "weight_zero_point"
)
# ---------------------------------------------------------------------------
# Dense DA8W4
# ---------------------------------------------------------------------------
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert kernel._maybe_process_da8w4_weights(layer)
@pytest.mark.parametrize(
"config_kwargs,reason",
[
({"act_type": torch.float32}, "f32 activations are rejected"),
(
{"weight_type": scalar_types.uint4, "zero_points": True},
"asymmetric int4 is unsupported",
),
],
)
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config(**config_kwargs))
assert not kernel._maybe_process_da8w4_weights(layer), reason
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"envs.VLLM_CPU_INT4_W4A8",
False,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
assert layer._zentorch_da8w4
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
assert layer._zentorch_da8w4_packed.dtype == torch.int8
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
assert layer._zentorch_da8w4_scale.shape == (
IN_FEATURES // GROUP_SIZE,
OUT_FEATURES,
)
assert layer._zentorch_da8w4_packed.is_contiguous()
assert layer._zentorch_da8w4_scale.is_contiguous()
# The checkpoint parameters are released once repacked.
assert layer.weight_packed.numel() == 0
assert layer.weight_scale.numel() == 0
def test_da8w4_apply_weights_matches_dequantized_reference(
mock_zentorch_ops, monkeypatch
):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, weight_deq = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
out = kernel.apply_weights(layer, x, bias)
expected = _dynamic_quant_matmul(x, weight_deq, bias)
assert out.dtype == torch.bfloat16
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: "zentorch_dynamic_qlinear" not in ops,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_can_implement_requires_zen_cpu(monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: False,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
assert not ok
assert reason is not None
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
CPUWNA16LinearKernel,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
"current_platform.is_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
unaligned = dataclasses.replace(
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
)
assert unaligned.partition_weight_shape[1] % 32 != 0
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
assert not ok, "the parent is expected to turn this shape away"
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
assert ok, reason
@pytest.mark.parametrize(
"group_size,expected_ok",
[
(-1, True), # per-channel, one group spanning K
(GROUP_SIZE, True),
(IN_FEATURES, True),
(6, False), # even, so the parent's multiple-of-2 rule would allow it
(0, False),
],
)
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(
_make_config(group_size=group_size)
)
assert ok is expected_ok, reason
if not expected_ok:
assert reason is not None
assert "must divide input size" in reason
# ---------------------------------------------------------------------------
# MoE DA8W4
# ---------------------------------------------------------------------------
NUM_EXPERTS = 4
HIDDEN = 256
INTERMEDIATE = 128
def _make_moe_weights(group_size: int = 32):
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
torch.manual_seed(0)
w13_q, w13_s, w2_q, w2_s = [], [], [], []
for _ in range(NUM_EXPERTS):
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
q13, s13 = _quantize_per_group(w13, group_size)
# CT is N-first and packs along the input dim: [N, K//8] per expert.
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
w13_s.append(s13)
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
q2, s2 = _quantize_per_group(w2, group_size)
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
w2_s.append(s2)
return (
torch.stack(w13_q),
torch.stack(w2_q),
torch.stack(w13_s),
torch.stack(w2_s),
)
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
backends = int_wna16._get_priority_backends()
assert backends[0] == WNA16MoEBackend.ZEN_CPU
# The generic CPU backend stays available as a fallback.
assert WNA16MoEBackend.CPU in backends
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
assert w13_out.dtype == w2_out.dtype == torch.int32
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
# Symmetric checkpoints carry no zero points into the kernel.
assert converted[8] is None and converted[9] is None
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
"""The repacked weight must dequantize back to the checkpoint values."""
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
expected = _import_unpack_from_int32()(
w13,
4,
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
packed_dim=1,
)
for expert in range(NUM_EXPERTS):
torch.testing.assert_close(
_unpack_s4(w13_out[expert], HIDDEN),
expected[expert].float(),
)
@pytest.mark.parametrize(
"group_size,expected_ok",
[(32, True), (128, True), (-1, False), (2, False)],
)
def test_zen_cpu_group_size_gating(group_size, expected_ok):
quant_config = type("Args", (), {"group_size": group_size})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
@pytest.mark.parametrize(
"may_have_zp,may_have_bias,expected_ok",
[(True, False, False), (False, True, True), (True, True, False)],
)
def test_zen_cpu_rejects_zero_points_but_takes_bias(
may_have_zp, may_have_bias, expected_ok
):
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
def test_zen_cpu_rejects_moe_wna16_layout():
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
quant_config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={},
)
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_disabled_by_env(monkeypatch):
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_backend_maps_to_experts_class():
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
ZentorchExpertsInt4
]
def test_zen_experts_support_predicates():
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
kInt4Static32,
kInt8StaticChannelSym,
)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
for act in MoEActivation:
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
assert ZentorchExpertsInt4._supports_activation(act) == expected
assert ZentorchExpertsInt4.requires_interleaved_w13
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
assert ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": False})()
)
assert not ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": True})()
)
def test_zen_experts_take_custom_routing():
"""Models with their own router (gemma-4) reach select_experts through the
config captured off the layer, since apply() cannot be handed the callable."""
from types import SimpleNamespace
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert ZentorchExpertsInt4._supports_routing_method(
RoutingMethodType.Custom, None, None
)
def routing_fn(**kwargs):
raise AssertionError("not called")
layer = _StubMoELayer(
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
MoEActivation.SILU,
)
layer.renormalize = True
layer.custom_routing_function = routing_fn
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
assert experts.custom_routing_function is routing_fn
assert experts.renormalize is True
def _swigluoai_perm(activation, experts_cls=None):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return swigluoai_w13_interleave_perm(
experts_cls or ZentorchExpertsInt4,
activation,
2 * INTERMEDIATE,
torch.device("cpu"),
)
def test_swigluoai_perm_only_for_zen_swigluoai():
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
# Other activations keep the half-split layout the loader leaves.
assert _swigluoai_perm(MoEActivation.SILU) is None
# object stands in for any kernel that never set requires_interleaved_w13.
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
"""The permutation is a gather on w13's output-channel axis, which is dim 1
for the N-first packed weights and their group scales."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
permuted = int_wna16._process_weights_zen_cpu(
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
)[0]
for expert in range(NUM_EXPERTS):
before = _unpack_s4(baseline[expert], HIDDEN)
after = _unpack_s4(permuted[expert], HIDDEN)
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])