[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,407 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import csv
|
||||
import importlib
|
||||
import importlib.util
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
|
||||
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
get_fp8_min_max,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_rocm()
|
||||
and current_platform.supports_fp8()
|
||||
and aiter_available
|
||||
),
|
||||
reason="Requires ROCm + FP8 support + aiter",
|
||||
),
|
||||
pytest.mark.usefixtures("default_vllm_config"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def enable_hipb_mm_kernel(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
yield
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _make_config(
|
||||
*,
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
out_dtype: torch.dtype = torch.bfloat16,
|
||||
weight_shape: tuple[int, int] = (512, 4096),
|
||||
) -> FP8ScaledMMLinearLayerConfig:
|
||||
return FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=kFp8DynamicTokenSym,
|
||||
weight_shape=weight_shape,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
|
||||
def _find_csv_row(path: str, m: int, n: int, k: int) -> dict | None:
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
with open(path, newline="") as f:
|
||||
reader = csv.DictReader(f, skipinitialspace=True)
|
||||
for row in reader:
|
||||
try:
|
||||
if (
|
||||
int(row.get("m", -1)) == m
|
||||
and int(row.get("n", -1)) == n
|
||||
and int(row.get("k", -1)) == k
|
||||
):
|
||||
return dict(row)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _skip_if_no_hipb_mm_solution(exc: RuntimeError) -> None:
|
||||
if "hipblasLtMatmulAlgoGetHeuristic found 0 valid solutions" in str(exc):
|
||||
pytest.skip(
|
||||
"hipb_mm bpreshuffle path has no valid hipBLASLt solution on "
|
||||
"this ROCm stack."
|
||||
)
|
||||
|
||||
|
||||
def _check_bpreshuffle_runtime_support(weight_shape: tuple[int, int], num_tokens: int):
|
||||
import aiter
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device="cuda")
|
||||
w = torch.randn(weight_shape, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
aiter.hipb_create_extension()
|
||||
x_q, x_scale = aiter.pertoken_quant(x, quant_dtype=current_platform.fp8_dtype())
|
||||
w_q, w_scale = aiter.pertoken_quant(w, quant_dtype=current_platform.fp8_dtype())
|
||||
|
||||
try:
|
||||
aiter.hipb_mm(
|
||||
x_q,
|
||||
shuffle_weight(w_q, layout=(16, 16)).t(),
|
||||
solution_index=-1,
|
||||
out_dtype=torch.bfloat16,
|
||||
scaleA=x_scale,
|
||||
scaleB=w_scale.t().contiguous(),
|
||||
scaleOut=None,
|
||||
bpreshuffle=True,
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
_skip_if_no_hipb_mm_solution(exc)
|
||||
raise
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_requires_hipbmm_flag(monkeypatch: pytest.MonkeyPatch):
|
||||
# The kernel rejects when `is_hip_fp8bmm_enabled()` is False. That helper
|
||||
# requires AITER + AITER_LINEAR + MI3xx, so dropping AITER_LINEAR exercises
|
||||
# the rejection branch.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "0")
|
||||
monkeypatch.delenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", raising=False)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
is_supported, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.is_supported()
|
||||
|
||||
assert not is_supported
|
||||
assert reason == (
|
||||
"requires setting `VLLM_ROCM_USE_AITER=1`, "
|
||||
"`VLLM_ROCM_USE_AITER_LINEAR=1`, "
|
||||
"and `VLLM_ROCM_USE_AITER_LINEAR_HIPBMM=1`."
|
||||
)
|
||||
|
||||
|
||||
def test_hipb_mm_flag_enables_hip_online_tuning(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
import vllm.envs as envs_mod
|
||||
import vllm.platforms.rocm as rocm_mod
|
||||
|
||||
# The rocm.py gate requires all three AITER flags (and MI3xx) to auto-set
|
||||
# HIP_ONLINE_TUNING.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
|
||||
|
||||
try:
|
||||
importlib.reload(envs_mod)
|
||||
importlib.reload(rocm_mod)
|
||||
assert envs_mod.VLLM_ROCM_USE_AITER
|
||||
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR
|
||||
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
|
||||
assert os.environ.get("HIP_ONLINE_TUNING") == "1"
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
os.environ.pop("HIP_ONLINE_TUNING", None)
|
||||
importlib.reload(envs_mod)
|
||||
importlib.reload(rocm_mod)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_can_implement_success(enable_hipb_mm_kernel):
|
||||
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
|
||||
_make_config()
|
||||
)
|
||||
|
||||
assert can_implement
|
||||
assert reason is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "expected_reason"),
|
||||
[
|
||||
(
|
||||
_make_config(weight_quant_key=kFp8StaticTensorSym),
|
||||
"requires per token activation scales and per channel weight scales.",
|
||||
),
|
||||
(
|
||||
_make_config(out_dtype=torch.float16),
|
||||
"requires bfloat16 output dtype.",
|
||||
),
|
||||
(
|
||||
_make_config(weight_shape=(8, 4090)),
|
||||
"requires N >= 16 and both N and K divisible by 16, "
|
||||
"received N=8 and K=4090.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hipb_mm_kernel_can_implement_rejects_unsupported_configs(
|
||||
enable_hipb_mm_kernel,
|
||||
config: FP8ScaledMMLinearLayerConfig,
|
||||
expected_reason: str,
|
||||
):
|
||||
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
|
||||
config
|
||||
)
|
||||
|
||||
assert not can_implement
|
||||
assert reason == expected_reason
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_process_weights_after_loading_shuffles_weights(
|
||||
enable_hipb_mm_kernel,
|
||||
):
|
||||
weight_shape = (512, 4096)
|
||||
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
|
||||
_make_config(weight_shape=weight_shape),
|
||||
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.rand(weight_shape, device="cuda").to(current_platform.fp8_dtype()).t(),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.rand((weight_shape[0], 1), dtype=torch.float32, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.input_scale = None
|
||||
layer.input_scale_ub = None
|
||||
|
||||
original_weight = layer.weight.detach().clone()
|
||||
original_weight_scale = layer.weight_scale.detach().clone()
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# process_weights_after_loading now pre-applies the transposes that used
|
||||
# to live in _rocm_aiter_hipb_mm_fp8_impl, so the stored weight is the
|
||||
# shuffled tensor with a trailing `.t()` view, and the stored weight scale
|
||||
# is its transposed-contiguous form.
|
||||
expected_weight = rocm_aiter_ops.shuffle_weight(
|
||||
original_weight.t().contiguous()
|
||||
).t()
|
||||
torch.testing.assert_close(layer.weight, expected_weight)
|
||||
|
||||
expected_weight_scale = original_weight_scale.t().contiguous()
|
||||
torch.testing.assert_close(layer.weight_scale, expected_weight_scale)
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel):
|
||||
import aiter
|
||||
|
||||
weight_shape = (512, 4096)
|
||||
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=32)
|
||||
|
||||
layer = TestFP8Layer(
|
||||
weight_shape=weight_shape,
|
||||
activation_quant_key=kFp8DynamicTokenSym,
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
# hipb_mm uses a transposed-result GEMM internally, so the flattened token
|
||||
# count becomes the effective N dimension passed into hipBLASLt. Keep it
|
||||
# aligned to avoid heuristic failures for tiny N.
|
||||
x = torch.randn(2, 16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
|
||||
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
try:
|
||||
out = layer(x, bias)
|
||||
except RuntimeError as exc:
|
||||
_skip_if_no_hipb_mm_solution(exc)
|
||||
raise
|
||||
|
||||
x_2d = x.view(-1, x.shape[-1])
|
||||
x_q, x_scale = layer.kernel.quant_fp8(
|
||||
x_2d,
|
||||
layer.input_scale,
|
||||
layer.input_scale_ub,
|
||||
)
|
||||
try:
|
||||
# process_weights_after_loading already applies the trailing `.t()` on
|
||||
# the shuffled weight and the `.t().contiguous()` on the weight scale,
|
||||
# so the raw aiter call uses them directly.
|
||||
expected = aiter.hipb_mm(
|
||||
x_q,
|
||||
layer.weight,
|
||||
solution_index=-1,
|
||||
bias=bias,
|
||||
out_dtype=torch.bfloat16,
|
||||
scaleA=x_scale,
|
||||
scaleB=layer.weight_scale,
|
||||
scaleOut=None,
|
||||
bpreshuffle=True,
|
||||
).view(*out.shape)
|
||||
except RuntimeError as exc:
|
||||
_skip_if_no_hipb_mm_solution(exc)
|
||||
raise
|
||||
|
||||
assert isinstance(layer.kernel, AiterHipbMMPerTokenFp8ScaledMMLinearKernel)
|
||||
assert out.shape == (2, 16, weight_shape[0])
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
|
||||
"""Kernel output should match a dequantized fp32 reference within
|
||||
fp8 per-token / per-channel quantization noise."""
|
||||
weight_shape = (512, 4096) # (N, K)
|
||||
num_tokens = 32
|
||||
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
fp8_max = get_fp8_min_max()[1]
|
||||
device = torch.device("cuda")
|
||||
|
||||
# Build a bf16 weight and quantize per output channel (one scale per row).
|
||||
w_bf16 = torch.randn(weight_shape, dtype=torch.bfloat16, device=device)
|
||||
w_amax = w_bf16.abs().amax(dim=1, keepdim=True).to(torch.float32)
|
||||
w_scale = (w_amax / fp8_max).clamp(min=1e-12)
|
||||
w_fp8 = (w_bf16.to(torch.float32) / w_scale).clamp(-fp8_max, fp8_max).to(fp8_dtype)
|
||||
w_dequant = w_fp8.to(torch.float32) * w_scale
|
||||
|
||||
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device=device)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
# Pre-`process_weights_after_loading` convention: weight stored as the
|
||||
# `[K, N]` view of the fp8 tensor.
|
||||
layer.weight = torch.nn.Parameter(w_fp8.t(), requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(w_scale, requires_grad=False)
|
||||
layer.input_scale = None
|
||||
layer.input_scale_ub = None
|
||||
|
||||
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
|
||||
_make_config(weight_shape=weight_shape),
|
||||
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device=device)
|
||||
|
||||
try:
|
||||
out = kernel.apply_weights(layer, x, bias)
|
||||
except RuntimeError as exc:
|
||||
_skip_if_no_hipb_mm_solution(exc)
|
||||
raise
|
||||
|
||||
# Reference: quantize x per-token the same way the kernel does, then run
|
||||
# the matmul in fp32 against the dequantized weight. This isolates plumbing
|
||||
# / reduction bugs from inherent fp8 quantization noise.
|
||||
x_amax = x.abs().amax(dim=1, keepdim=True).to(torch.float32)
|
||||
x_scale_ref = (x_amax / fp8_max).clamp(min=1e-12)
|
||||
x_q = (x.to(torch.float32) / x_scale_ref).clamp(-fp8_max, fp8_max).to(fp8_dtype)
|
||||
x_dequant = x_q.to(torch.float32) * x_scale_ref
|
||||
expected = (x_dequant @ w_dequant.t() + bias.to(torch.float32)).to(torch.bfloat16)
|
||||
|
||||
assert out.shape == (num_tokens, weight_shape[0])
|
||||
# K=4096 fp8 reduction leaves room for accumulation order drift and
|
||||
# catastrophic cancellation on near-zero outputs; tolerances are loose
|
||||
# enough to absorb that but tight enough to catch wrong layouts, missing
|
||||
# bias, swapped scales, etc.
|
||||
torch.testing.assert_close(out, expected, atol=5.0, rtol=0.1)
|
||||
|
||||
|
||||
def test_hipb_mm_kernel_online_tuning_writes_csv(
|
||||
enable_hipb_mm_kernel,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path,
|
||||
):
|
||||
weight_shape = (256, 4096)
|
||||
cache_file = tmp_path / "hip_online_tuning_res.csv"
|
||||
|
||||
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=16)
|
||||
|
||||
monkeypatch.setenv("HIP_ONLINE_TUNING", "1")
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
layer = TestFP8Layer(
|
||||
weight_shape=weight_shape,
|
||||
activation_quant_key=kFp8DynamicTokenSym,
|
||||
weight_quant_key=kFp8StaticChannelSym,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
# The effective heuristic N dimension is the flattened token count.
|
||||
x = torch.randn(16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
|
||||
try:
|
||||
out = layer(x)
|
||||
except RuntimeError as exc:
|
||||
_skip_if_no_hipb_mm_solution(exc)
|
||||
raise
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert out.shape == (16, weight_shape[0])
|
||||
assert cache_file.exists()
|
||||
|
||||
# hipb_mm records the internal GEMM dimensions used by hipBLASLt after its
|
||||
# transposed-result transformation.
|
||||
row = _find_csv_row(
|
||||
str(cache_file),
|
||||
m=weight_shape[0],
|
||||
n=x.shape[0],
|
||||
k=weight_shape[1],
|
||||
)
|
||||
assert row is not None
|
||||
@@ -1,29 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm import _custom_ops as ops # noqa: F401
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops._C, "awq_dequantize"),
|
||||
reason="AWQ is not supported on this GPU type.",
|
||||
)
|
||||
def test_awq_dequantize_opcheck(monkeypatch: pytest.MonkeyPatch):
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_USE_TRITON_AWQ", "0")
|
||||
qweight = torch.randint(
|
||||
-2000000000, 2000000000, (8192, 256), device="cuda", dtype=torch.int32
|
||||
)
|
||||
scales = torch.rand((64, 2048), device="cuda", dtype=torch.float16)
|
||||
zeros = torch.empty((64, 256), device="cuda", dtype=torch.int32)
|
||||
split_k_iters = 0
|
||||
thx = 0
|
||||
thy = 0
|
||||
opcheck(
|
||||
torch.ops._C.awq_dequantize,
|
||||
(qweight, scales, zeros, split_k_iters, thx, thy),
|
||||
)
|
||||
@@ -1,177 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AWQ Triton kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_awq_triton.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.awq_triton import (
|
||||
AWQ_TRITON_SUPPORTED_GROUP_SIZES,
|
||||
awq_dequantize_triton,
|
||||
awq_gemm_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="AWQ Triton kernels require CUDA/ROCm or XPU.",
|
||||
)
|
||||
|
||||
device = current_platform.device_type
|
||||
|
||||
|
||||
def reverse_awq_order(t: torch.Tensor):
|
||||
bits = 4
|
||||
AWQ_REVERSE_ORDER = [0, 4, 1, 5, 2, 6, 3, 7]
|
||||
reverse_order_tensor = torch.arange(
|
||||
t.shape[-1],
|
||||
dtype=torch.int32,
|
||||
device=t.device,
|
||||
)
|
||||
reverse_order_tensor = reverse_order_tensor.view(-1, 32 // bits)
|
||||
reverse_order_tensor = reverse_order_tensor[:, AWQ_REVERSE_ORDER]
|
||||
reverse_order_tensor = reverse_order_tensor.view(-1)
|
||||
|
||||
t = t[:, reverse_order_tensor] & 0xF
|
||||
return t
|
||||
|
||||
|
||||
# qweights - [R , C // 8], int32
|
||||
# scales - [R // G, C ], float16
|
||||
# zeros - [R // G, C // 8], int32
|
||||
def awq_dequantize_torch(
|
||||
qweight: torch.Tensor, scales: torch.Tensor, qzeros: torch.Tensor, group_size: int
|
||||
) -> torch.Tensor:
|
||||
if group_size == -1:
|
||||
group_size = qweight.shape[0]
|
||||
|
||||
bits = 4
|
||||
shifts = torch.arange(0, 32, bits, device=qzeros.device)
|
||||
|
||||
iweights = torch.bitwise_right_shift(qweight[:, :, None], shifts[None, None, :]).to(
|
||||
torch.int8
|
||||
)
|
||||
|
||||
iweights = iweights.view(iweights.shape[0], -1)
|
||||
|
||||
zeros = torch.bitwise_right_shift(qzeros[:, :, None], shifts[None, None, :]).to(
|
||||
torch.int8
|
||||
)
|
||||
zeros = zeros.view(qzeros.shape[0], -1)
|
||||
zeros = reverse_awq_order(zeros)
|
||||
|
||||
iweights = reverse_awq_order(iweights)
|
||||
|
||||
iweights = torch.bitwise_and(iweights, (2**bits) - 1)
|
||||
zeros = torch.bitwise_and(zeros, (2**bits) - 1)
|
||||
|
||||
scales = scales.repeat_interleave(group_size, dim=0)
|
||||
zeros = zeros.repeat_interleave(group_size, dim=0)
|
||||
return (iweights - zeros) * scales
|
||||
|
||||
|
||||
# qweights - [R , C // 8], int32
|
||||
# scales - [R // G, C ], float16
|
||||
# zeros - [R // G, C // 8], int32
|
||||
@pytest.mark.parametrize("qweight_rows", [3584, 18944, 128, 256, 512, 1024])
|
||||
@pytest.mark.parametrize("qweight_cols", [448, 576, 4736, 16, 32, 64, 128])
|
||||
@pytest.mark.parametrize("group_size", AWQ_TRITON_SUPPORTED_GROUP_SIZES)
|
||||
def test_dequantize(qweight_rows, qweight_cols, group_size):
|
||||
if group_size == -1:
|
||||
group_size = qweight_rows
|
||||
|
||||
qweight_dtype = torch.int32
|
||||
scales_rows = qweight_rows // group_size
|
||||
scales_cols = qweight_cols * 8
|
||||
scales_dtype = torch.float16
|
||||
zeros_rows = scales_rows
|
||||
zeros_cols = qweight_cols
|
||||
zeros_dtype = torch.int32
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
qweight = torch.randint(
|
||||
0,
|
||||
torch.iinfo(torch.int32).max,
|
||||
(qweight_rows, qweight_cols),
|
||||
dtype=qweight_dtype,
|
||||
device=device,
|
||||
)
|
||||
scales = torch.rand(scales_rows, scales_cols, dtype=scales_dtype, device=device)
|
||||
zeros = torch.randint(
|
||||
0,
|
||||
torch.iinfo(torch.int32).max,
|
||||
(zeros_rows, zeros_cols),
|
||||
dtype=zeros_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
iweights_triton = awq_dequantize_triton(qweight, scales, zeros)
|
||||
|
||||
assert not torch.any(torch.isinf(iweights_triton)) and not torch.any(
|
||||
torch.isnan(iweights_triton)
|
||||
)
|
||||
|
||||
iweights_torch = awq_dequantize_torch(qweight, scales, zeros, group_size)
|
||||
|
||||
torch.testing.assert_close(iweights_triton, iweights_torch)
|
||||
|
||||
|
||||
# input - [N, K]
|
||||
# qweight - [K, M // 8]
|
||||
# qzeros - [K // G, M // 8]
|
||||
# scales - [K // G, M]
|
||||
@pytest.mark.parametrize("N", [1, 2, 4, 8, 14, 17, 23, 32])
|
||||
@pytest.mark.parametrize("K", [128])
|
||||
@pytest.mark.parametrize("M", [16, 24, 32])
|
||||
@pytest.mark.parametrize("group_size", AWQ_TRITON_SUPPORTED_GROUP_SIZES)
|
||||
@pytest.mark.parametrize("splitK", [1, 8])
|
||||
def test_gemm(N, K, M, splitK, group_size):
|
||||
if group_size == -1:
|
||||
group_size = K
|
||||
|
||||
split_k_iters = splitK
|
||||
|
||||
input_rows = N
|
||||
input_cols = K
|
||||
input_dtype = torch.float32
|
||||
qweight_rows = input_cols
|
||||
qweight_cols = M // 8
|
||||
scales_rows = qweight_rows // group_size
|
||||
scales_cols = M
|
||||
scales_dtype = torch.float32
|
||||
qzeros_rows = scales_rows
|
||||
qzeros_cols = qweight_cols
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
input = torch.rand((input_rows, input_cols), dtype=input_dtype, device=device)
|
||||
qweight = torch.randint(
|
||||
0, torch.iinfo(torch.int32).max, (qweight_rows, qweight_cols), device=device
|
||||
)
|
||||
qzeros = torch.randint(
|
||||
0, torch.iinfo(torch.int32).max, (qzeros_rows, qzeros_cols), device=device
|
||||
)
|
||||
scales = torch.rand((scales_rows, scales_cols), dtype=scales_dtype, device=device)
|
||||
|
||||
output_triton = awq_gemm_triton(input, qweight, scales, qzeros, split_k_iters)
|
||||
|
||||
assert not torch.any(torch.isinf(output_triton)) and not torch.any(
|
||||
torch.isnan(output_triton)
|
||||
)
|
||||
|
||||
dequantized_weights = awq_dequantize_triton(qweight, scales, qzeros)
|
||||
|
||||
output_torch = torch.matmul(input, dequantized_weights)
|
||||
|
||||
assert not torch.any(torch.isinf(output_torch)) and not torch.any(
|
||||
torch.isnan(output_torch)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output_triton.cpu(), output_torch.cpu(), atol=1e-1, rtol=1e-1
|
||||
)
|
||||
@@ -1,423 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from https://github.com/sgl-project/sglang/pull/2575
|
||||
import itertools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quant_utils import (
|
||||
native_per_token_group_quant_fp8,
|
||||
native_w8a8_block_matmul,
|
||||
)
|
||||
from tests.kernels.utils import fp8_ulp_distance
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.b12x import (
|
||||
B12xFp8BlockScaledMMKernel,
|
||||
_run_b12x_fp8_block_scaled_mm,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import cutlass_scaled_mm
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
w8a8_triton_block_scaled_mm,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
get_tma_aligned_size,
|
||||
is_deep_gemm_e8m0_used,
|
||||
per_block_cast_to_fp8,
|
||||
should_use_deepgemm_for_fp8_linear,
|
||||
)
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_fp8_blockscale_gemm,
|
||||
has_flashinfer_fp8_blockscale_gemm,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability < (9, 0):
|
||||
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
# Test configurations
|
||||
DTYPES = [torch.bfloat16] # [torch.half, torch.bfloat16, torch.float32]
|
||||
# Quantization test configs
|
||||
NUM_TOKENS = [7, 2050]
|
||||
D = [512, 4096, 5120, 13824]
|
||||
GROUP_SIZE = [64, 128, 512]
|
||||
COLUMN_MAJOR_SCALES = [True, False]
|
||||
TMA_ALIGNED_SCALES = [True, False]
|
||||
# Matmul test configs
|
||||
M = [1, 7, 8, 83, 4096]
|
||||
N = [128, 512, 576, 7168, 13824]
|
||||
K = [256, 3884, 4096, 13824, 16384]
|
||||
# Deepseek-V3's intermediate size 18432, so N is 18432*2/8=4608 at TP8
|
||||
# and its hidden size is 7168.
|
||||
BLOCK_SIZE = [[128, 128]]
|
||||
OUT_DTYPES = [torch.bfloat16] # [torch.float32, torch.half, torch.bfloat16]
|
||||
SEEDS = [0]
|
||||
|
||||
# Skip all tests if CUDA is not available
|
||||
pytest.importorskip("torch.cuda")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_fp8_fnuz(),
|
||||
reason="This platform supports e4m3fnuz, not e4m3fn.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,d,dtype,group_size,column_major_scales,tma_aligned_scales,seed",
|
||||
itertools.product(
|
||||
NUM_TOKENS,
|
||||
D,
|
||||
DTYPES,
|
||||
GROUP_SIZE,
|
||||
COLUMN_MAJOR_SCALES,
|
||||
TMA_ALIGNED_SCALES,
|
||||
SEEDS,
|
||||
),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_per_token_group_quant_fp8(
|
||||
num_tokens, d, dtype, group_size, column_major_scales, tma_aligned_scales, seed
|
||||
):
|
||||
torch.manual_seed(seed)
|
||||
x = torch.rand(num_tokens, d, dtype=dtype)
|
||||
|
||||
ref_out, ref_scale = native_per_token_group_quant_fp8(x, group_size)
|
||||
out, scale = per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
column_major_scales=column_major_scales,
|
||||
tma_aligned_scales=tma_aligned_scales,
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# On gfx950 the Triton and PyTorch FP8 kernels can round in opposite
|
||||
# directions when an element lands at the midpoint between two adjacent
|
||||
# e4m3fn values (1-ULP tie-breaking). Verify: (1) no element is more
|
||||
# than 1 FP8 ULP away, and (2) fewer than 0.05% of elements have any
|
||||
# mismatch. Observed worst case across all parameter combos: 0.049%,
|
||||
# max ULP = 1.
|
||||
ulp = fp8_ulp_distance(out, ref_out)
|
||||
assert (ulp <= 1).all(), (
|
||||
f"FP8 mismatch > 1 ULP: {int((ulp > 1).sum())} elements"
|
||||
)
|
||||
assert float((ulp > 0).float().mean()) < 5e-4, (
|
||||
f"Too many 1-ULP mismatches: {int((ulp > 0).sum())}/{ulp.numel()}"
|
||||
)
|
||||
else:
|
||||
assert torch.allclose(
|
||||
out.to(torch.float32), ref_out.to(torch.float32), rtol=0.15
|
||||
)
|
||||
assert torch.allclose(scale, ref_scale)
|
||||
|
||||
if column_major_scales:
|
||||
assert scale.stride()[-2] == 1
|
||||
if tma_aligned_scales:
|
||||
assert scale.stride()[-1] == get_tma_aligned_size(num_tokens, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,N,K,block_size,out_dtype,seed",
|
||||
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_matmul(M, N, K, block_size, out_dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
factor_for_scale = 1e-2
|
||||
fp8_info = torch.finfo(current_platform.fp8_dtype())
|
||||
fp8_max, fp8_min = fp8_info.max, fp8_info.min
|
||||
|
||||
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
A_fp8 = A_fp32.clamp(min=fp8_min, max=fp8_max).to(current_platform.fp8_dtype())
|
||||
|
||||
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(current_platform.fp8_dtype())
|
||||
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n_tiles = (N + block_n - 1) // block_n
|
||||
k_tiles = (K + block_k - 1) // block_k
|
||||
|
||||
As = torch.rand(M, k_tiles, dtype=torch.float32) * factor_for_scale
|
||||
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
|
||||
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
out = w8a8_triton_block_scaled_mm(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUTLASS only supported on CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
# 65/66/67 cover all M%4 residue classes above the SM100 swapAB
|
||||
# threshold (m <= 64); 1026 crosses multiple 128-row SF atoms.
|
||||
"M",
|
||||
[32, 65, 66, 67, 1026],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_cutlass_matmul(M):
|
||||
# Test simple case where weight.shape % 128 != 0,
|
||||
# like in DSV3 kv_a_proj_with_mqa
|
||||
N = 576
|
||||
K = 7168
|
||||
block_size = [128, 128]
|
||||
out_dtype = torch.bfloat16
|
||||
seed = 0
|
||||
|
||||
torch.manual_seed(seed)
|
||||
factor_for_scale = 1e-2
|
||||
fp8_info = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max, fp8_min = fp8_info.max, fp8_info.min
|
||||
|
||||
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
|
||||
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n_tiles = (N + block_n - 1) // block_n
|
||||
k_tiles = (K + block_k - 1) // block_k
|
||||
|
||||
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
|
||||
|
||||
A_fp8, As = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=False
|
||||
)
|
||||
# CUTLASS uses column-major format for scales
|
||||
A_fp8_cutlass, As_cutlass = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=True
|
||||
)
|
||||
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
out = cutlass_scaled_mm(A_fp8_cutlass, B_fp8, As_cutlass, Bs, block_size, out_dtype)
|
||||
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() and current_platform.has_device_capability(90)),
|
||||
reason="torch._scaled_mm DeepSeek-style block scaling only supports SM90.",
|
||||
)
|
||||
def test_w8a8_block_fp8_torch_scaled_mm_matmul():
|
||||
# BlockWiseTorchFP8ScaledMMLinearKernel: 1x128 activation + 128x128 weight
|
||||
# block scaling routed through torch._scaled_mm. M=83 is not a multiple of
|
||||
# 4 so this also exercises the M-padding path.
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.pytorch import (
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
M = 83
|
||||
N = 576
|
||||
K = 7168
|
||||
block_size = [128, 128]
|
||||
out_dtype = torch.bfloat16
|
||||
seed = 0
|
||||
|
||||
torch.manual_seed(seed)
|
||||
factor_for_scale = 1e-2
|
||||
fp8_info = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max, fp8_min = fp8_info.max, fp8_info.min
|
||||
|
||||
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp8 = B_fp32.clamp(min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
block_n, block_k = block_size[0], block_size[1]
|
||||
n_tiles = (N + block_n - 1) // block_n
|
||||
k_tiles = (K + block_k - 1) // block_k
|
||||
Bs = torch.rand(n_tiles, k_tiles, dtype=torch.float32) * factor_for_scale
|
||||
|
||||
# Reference uses row-major activation scales.
|
||||
A_fp8, As = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=False
|
||||
)
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
|
||||
# The kernel expects column-major activation scales on CUDA.
|
||||
A_fp8_cuda, As_cuda = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=True
|
||||
)
|
||||
|
||||
stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__(
|
||||
BlockWiseTorchFP8ScaledMMLinearKernel
|
||||
)
|
||||
stub.config = types.SimpleNamespace(out_dtype=out_dtype)
|
||||
out = stub.apply_block_scaled_mm(
|
||||
A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda()
|
||||
)
|
||||
|
||||
ref_out = ref_out.cuda()
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_fp8_fnuz(),
|
||||
reason="This platform supports e4m3fnuz, not e4m3fn.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"M,N,K,block_size,out_dtype,seed",
|
||||
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
|
||||
)
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGemm kernels not available.")
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_deep_gemm_matmul(M, N, K, block_size, out_dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
fp8_info = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max = fp8_info.max
|
||||
|
||||
A_fp32 = (torch.rand(M, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
B_fp32 = (torch.rand(N, K, dtype=torch.float32) - 0.5) * 2 * fp8_max
|
||||
|
||||
# only aligned sizes are supported by deepgemm
|
||||
if not should_use_deepgemm_for_fp8_linear(
|
||||
output_dtype=out_dtype, weight_shape=B_fp32.shape, supports_deep_gemm=True
|
||||
):
|
||||
pytest.skip(f"Skipping test; invalid size {M}, {N}, {K}")
|
||||
|
||||
A_fp8, As_fp8 = per_token_group_quant_fp8(
|
||||
A_fp32, block_size[1], column_major_scales=True, tma_aligned_scales=True
|
||||
)
|
||||
B_fp8, Bs_fp8 = per_block_cast_to_fp8(
|
||||
B_fp32,
|
||||
block_size=block_size,
|
||||
use_ue8m0=is_deep_gemm_e8m0_used(),
|
||||
)
|
||||
|
||||
As = As_fp8.to(torch.float32)
|
||||
Bs = Bs_fp8.to(torch.float32)
|
||||
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
|
||||
out = torch.zeros((M, N), device="cuda", dtype=out_dtype)
|
||||
|
||||
assert As_fp8.shape == (M, (K + 127) // 128), (
|
||||
f"{As_fp8.shape} != {(M, (K + 127) // 128)}"
|
||||
)
|
||||
|
||||
fp8_gemm_nt((A_fp8, As_fp8), (B_fp8, Bs_fp8), out)
|
||||
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_fp8_fnuz(),
|
||||
reason="This platform supports e4m3fnuz, not e4m3fn.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"M,N,K,block_size,out_dtype,seed",
|
||||
itertools.product(M, N, K, BLOCK_SIZE, OUT_DTYPES, SEEDS),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_flashinfer_matmul(M, N, K, block_size, out_dtype, seed):
|
||||
if not has_flashinfer_fp8_blockscale_gemm():
|
||||
pytest.skip(
|
||||
"FlashInfer block GEMM not available (requires SM90+ and FlashInfer)"
|
||||
)
|
||||
# only aligned sizes
|
||||
if K % 128 != 0 or N % 64 != 0:
|
||||
pytest.skip(f"Skipping test; invalid size {M}, {N}, {K}")
|
||||
|
||||
torch.manual_seed(seed)
|
||||
fp8_info = torch.finfo(torch.float8_e4m3fn)
|
||||
fp8_max = fp8_info.max
|
||||
|
||||
A_bf16 = (torch.rand(M, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
|
||||
B_bf16 = (torch.rand(N, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
|
||||
|
||||
A_fp8, As_fp8 = per_token_group_quant_fp8(A_bf16, block_size[1], use_ue8m0=False)
|
||||
B_fp8, Bs_fp8 = per_block_cast_to_fp8(B_bf16, block_size, use_ue8m0=False)
|
||||
|
||||
As = As_fp8.to(torch.float32)
|
||||
Bs = Bs_fp8.to(torch.float32)
|
||||
|
||||
ref_out = native_w8a8_block_matmul(A_fp8, B_fp8, As, Bs, block_size, out_dtype)
|
||||
|
||||
out = flashinfer_fp8_blockscale_gemm(
|
||||
input=A_bf16,
|
||||
weight=B_fp8,
|
||||
input_scale=None,
|
||||
weight_scale=Bs,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.bfloat16) - ref_out.to(torch.bfloat16))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.bfloat16)))
|
||||
assert rel_diff < 0.001
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,N,K",
|
||||
[(1, 128, 256), (8, 256, 512), (129, 256, 256), (2, 4096, 4096)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_fp8_b12x_matmul(M, N, K):
|
||||
supported, reason = B12xFp8BlockScaledMMKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
torch.manual_seed(M)
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
A_bf16 = (torch.rand(M, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
|
||||
B_bf16 = (torch.rand(N, K, dtype=torch.bfloat16) - 0.5) * 2 * fp8_max
|
||||
A_fp8, As = per_token_group_quant_fp8(A_bf16, 128, use_ue8m0=False)
|
||||
B_fp8, Bs = per_block_cast_to_fp8(
|
||||
B_bf16,
|
||||
block_size=[128, 128],
|
||||
use_ue8m0=False,
|
||||
)
|
||||
As = As.float()
|
||||
Bs = Bs.float()
|
||||
|
||||
ref_out = native_w8a8_block_matmul(
|
||||
A_fp8,
|
||||
B_fp8,
|
||||
As,
|
||||
Bs,
|
||||
[128, 128],
|
||||
torch.bfloat16,
|
||||
)
|
||||
out = _run_b12x_fp8_block_scaled_mm(
|
||||
A_fp8,
|
||||
B_fp8,
|
||||
As,
|
||||
Bs,
|
||||
torch.bfloat16,
|
||||
)
|
||||
|
||||
rel_diff = torch.mean(torch.abs(out.float() - ref_out.float())) / torch.mean(
|
||||
torch.abs(ref_out.float())
|
||||
)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
out.float().flatten(),
|
||||
ref_out.float().flatten(),
|
||||
dim=0,
|
||||
)
|
||||
# Four-way split-K uses atomic BF16 reductions, so nondeterministic atomic
|
||||
# ordering can flap an output between adjacent BF16 values one ULP apart.
|
||||
assert rel_diff < 0.003
|
||||
assert cosine >= 0.99999
|
||||
@@ -1,497 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU FP8 scaled-mm GEMM kernels.
|
||||
|
||||
Covers:
|
||||
* fp8_scaled_mm_cpu (W8A16)
|
||||
* fp8_scaled_mm_with_quant (W8A8)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_cpu_fp8_scaled_mm.py -v`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cpu import (
|
||||
CPUFp8PerTensorScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
requires_cpu_fp8_w8a16 = pytest.mark.skipif(
|
||||
not ops._supports_cpu_fp8_w8a16, reason="fp8_scaled_mm_cpu op not available"
|
||||
)
|
||||
requires_cpu_fp8_w8a8 = pytest.mark.skipif(
|
||||
not ops.cpu_has_amx_fp8(),
|
||||
reason="requires native AMX-FP8 hardware",
|
||||
)
|
||||
|
||||
if not torch.cpu._is_amx_tile_supported():
|
||||
pytest.skip("requires AMX tile support", allow_module_level=True)
|
||||
|
||||
FP8_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
BLOCK_SIZE = [128, 128]
|
||||
GROUP_SIZE = 128
|
||||
|
||||
|
||||
def cdiv(a: int, b: int) -> int:
|
||||
return -(a // -b)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A16 helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quantize_weight_block_fp8(
|
||||
weight: torch.Tensor,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with block scales.
|
||||
|
||||
Returns:
|
||||
fp8_weight: [N, K] float8_e4m3fn
|
||||
scales: [n_tiles, k_tiles] float32
|
||||
|
||||
"""
|
||||
N, K = weight.shape
|
||||
block_n, block_k = block_size
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
n_tiles = cdiv(N, block_n)
|
||||
k_tiles = cdiv(K, block_k)
|
||||
|
||||
# Pad for even blocking
|
||||
pad_N = (block_n - (N % block_n)) % block_n
|
||||
pad_K = (block_k - (K % block_k)) % block_k
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
weight = torch.nn.functional.pad(weight, (0, pad_K, 0, pad_N))
|
||||
|
||||
# Reshape into blocks
|
||||
w_blocks = weight.view(n_tiles, block_n, k_tiles, block_k)
|
||||
w_blocks = w_blocks.permute(0, 2, 1, 3).contiguous()
|
||||
|
||||
# Per-block scale
|
||||
abs_max = w_blocks.abs().amax(dim=(-2, -1), keepdim=True)
|
||||
scales = abs_max / fp8_max
|
||||
scales = torch.where(scales == 0, torch.ones_like(scales), scales)
|
||||
|
||||
# Quantize
|
||||
q_fp8 = (w_blocks / scales).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
|
||||
# Reshape back
|
||||
fp8_weight = (
|
||||
q_fp8.permute(0, 2, 1, 3)
|
||||
.contiguous()
|
||||
.view(N + pad_N, K + pad_K)[:N, :K]
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
scales = scales.view(n_tiles, k_tiles)
|
||||
return fp8_weight, scales
|
||||
|
||||
|
||||
def dequant_weight_block_fp8(
|
||||
fp8_weight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
block_size: list[int],
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize FP8 weight back to float for reference computation."""
|
||||
N, K = fp8_weight.shape
|
||||
block_n, block_k = block_size
|
||||
n_tiles, k_tiles = scales.shape
|
||||
|
||||
pad_N = (block_n - (N % block_n)) % block_n
|
||||
pad_K = (block_k - (K % block_k)) % block_k
|
||||
if pad_N > 0 or pad_K > 0:
|
||||
fp8_padded = torch.nn.functional.pad(fp8_weight.float(), (0, pad_K, 0, pad_N))
|
||||
else:
|
||||
fp8_padded = fp8_weight.float()
|
||||
|
||||
w_blocks = fp8_padded.view(n_tiles, block_n, k_tiles, block_k)
|
||||
w_blocks = w_blocks.permute(0, 2, 1, 3).contiguous()
|
||||
dq = w_blocks * scales.view(n_tiles, k_tiles, 1, 1)
|
||||
dq = dq.permute(0, 2, 1, 3).contiguous().view(N + pad_N, K + pad_K)
|
||||
return dq[:N, :K].to(out_dtype)
|
||||
|
||||
|
||||
def ref_fp8_block_scaled_mm(
|
||||
x: torch.Tensor,
|
||||
fp8_weight: torch.Tensor,
|
||||
scales: torch.Tensor,
|
||||
block_size: list[int],
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Reference: dequant FP8→float32, matmul in float32, cast to out_dtype."""
|
||||
w_dq = dequant_weight_block_fp8(fp8_weight, scales, block_size, torch.float32)
|
||||
out = torch.mm(x.float(), w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A16 test: fp8_scaled_mm_cpu
|
||||
# ---------------------------------------------------------------------------
|
||||
M_SIZES = [1, 4, 16, 64, 128]
|
||||
# (N, K) — weight shape is [N, K], output has N columns.
|
||||
NK_SIZES = [
|
||||
(128, 256),
|
||||
(256, 512),
|
||||
(512, 1024),
|
||||
(1024, 2048),
|
||||
(5120, 5120),
|
||||
(17408, 5120),
|
||||
(5120, 17408),
|
||||
]
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a16
|
||||
@pytest.mark.parametrize("M", M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_scaled_mm(M: int, N: int, K: int, use_bias: bool):
|
||||
"""fp8_scaled_mm_cpu correctness against float reference."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
block_size = BLOCK_SIZE
|
||||
|
||||
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
|
||||
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
|
||||
fp8_weight, scales = quantize_weight_block_fp8(w_f32, block_size)
|
||||
|
||||
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
|
||||
|
||||
ref_out = ref_fp8_block_scaled_mm(
|
||||
x, fp8_weight, scales, block_size, bias, out_dtype
|
||||
)
|
||||
|
||||
packed_weight = torch.ops._C.convert_weight_packed(fp8_weight)
|
||||
kernel_out = ops.fp8_scaled_mm_cpu(
|
||||
x,
|
||||
packed_weight,
|
||||
scales,
|
||||
block_size,
|
||||
bias,
|
||||
out_dtype,
|
||||
True,
|
||||
)
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
|
||||
|
||||
|
||||
def quantize_weight_per_tensor_fp8(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with a single per-tensor scale."""
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
scale = weight.abs().amax() / fp8_max
|
||||
scale = torch.where(scale == 0, torch.ones_like(scale), scale)
|
||||
q = (weight / scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
return q, scale
|
||||
|
||||
|
||||
def ref_fp8_per_tensor_scaled_mm(
|
||||
x: torch.Tensor,
|
||||
fp8_weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
w_dq = fp8_weight.float() * scale
|
||||
out = torch.mm(x.float(), w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A8 helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quantize_weight_per_tensor(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with a single per-tensor scale, broadcast
|
||||
to [N, 1] (the shape float8_linear_prepack_cpu expects)."""
|
||||
N = weight.shape[0]
|
||||
abs_max = weight.abs().max()
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q, scale.expand(N).contiguous().view(N, 1)
|
||||
|
||||
|
||||
def quantize_weight_per_channel(
|
||||
weight: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with per-channel (per-row) scales [N, 1]."""
|
||||
abs_max = weight.abs().amax(dim=1, keepdim=True)
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (weight.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q, scale.float()
|
||||
|
||||
|
||||
def quantize_weight_per_group(
|
||||
weight: torch.Tensor,
|
||||
group_size: int = GROUP_SIZE,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Quantize weight [N, K] to FP8 with per-group (128-block) scales [N, G]."""
|
||||
N, K = weight.shape
|
||||
G = K // group_size
|
||||
w_groups = weight.view(N, G, group_size)
|
||||
abs_max = w_groups.abs().amax(dim=2, keepdim=True)
|
||||
scale = (abs_max / FP8_MAX).clamp(min=1e-7)
|
||||
q = (w_groups.float() / scale).clamp(-FP8_MAX, FP8_MAX).to(torch.float8_e4m3fn)
|
||||
return q.view(N, K), scale.squeeze(2).float()
|
||||
|
||||
|
||||
def quantize_act_per_token(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Reference per-token (row) dynamic FP8 quantization of activations."""
|
||||
x_f = x.float()
|
||||
scale = (x_f.abs().amax(dim=1, keepdim=True) / FP8_MAX).clamp(min=1e-7)
|
||||
scaled = (x_f / scale).clamp(-FP8_MAX, FP8_MAX)
|
||||
# Round via fp16 to match the AVX10.2 kernel path.
|
||||
q = scaled.half().to(torch.float8_e4m3fn)
|
||||
return q, scale
|
||||
|
||||
|
||||
def reference_static_fp8_linear(
|
||||
x: torch.Tensor, # BF16 [M, K]
|
||||
weight_fp8: torch.Tensor, # FP8 [N, K]
|
||||
weight_scale: torch.Tensor, # float32 [N, 1]
|
||||
act_scale: torch.Tensor, # float32 scalar
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Static per-tensor: quantize/dequant x with act_scale, dequant weight, matmul."""
|
||||
x_scaled = (x.float() / act_scale.item()).clamp(-FP8_MAX, FP8_MAX)
|
||||
# Round via fp16 to match the AVX10.2 kernel path.
|
||||
x_q = x_scaled.half().to(torch.float8_e4m3fn)
|
||||
x_dq = x_q.float() * act_scale.item()
|
||||
w_dq = weight_fp8.float() * weight_scale.view(-1, 1)
|
||||
out = torch.mm(x_dq, w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
NK_SIZES_PER_TENSOR = [
|
||||
(32, 64),
|
||||
(5120, 5120),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 64])
|
||||
@pytest.mark.parametrize("N,K", NK_SIZES_PER_TENSOR)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_per_tensor_scaled_mm_kernel(
|
||||
M: int, N: int, K: int, use_bias: bool, default_vllm_config
|
||||
):
|
||||
"""CPUFp8PerTensorScaledMMLinearKernel correctness against float reference.
|
||||
|
||||
Exercises the full kernel class (not just the raw op), including the
|
||||
weight-orientation fixup: Fp8LinearMethod stores `layer.weight` as
|
||||
[K, N] (torch._scaled_mm convention) before calling
|
||||
process_weights_after_loading, so the kernel must transpose back to
|
||||
[N, K] before VNNI-packing.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=out_dtype) / (K**0.5)
|
||||
w_f32 = torch.randn(N, K, dtype=torch.float32) / (K**0.5)
|
||||
fp8_weight, scale = quantize_weight_per_tensor_fp8(w_f32)
|
||||
bias = torch.randn(N, dtype=torch.float32) * 0.1 if use_bias else None
|
||||
|
||||
ref_out = ref_fp8_per_tensor_scaled_mm(x, fp8_weight, scale, bias, out_dtype)
|
||||
|
||||
config = FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_shape=(N, K),
|
||||
input_dtype=out_dtype,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
kernel = CPUFp8PerTensorScaledMMLinearKernel(
|
||||
config,
|
||||
layer_param_names=["weight", "weight_scale", "input_scale", "input_scale_ub"],
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
# Fp8LinearMethod stores the weight transposed to [K, N] before calling
|
||||
# process_weights_after_loading.
|
||||
layer.register_parameter(
|
||||
"weight", torch.nn.Parameter(fp8_weight.t().contiguous(), requires_grad=False)
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(scale.clone(), requires_grad=False)
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
kernel_out = kernel.apply_weights(layer, x, bias)
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.02, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [16, 48])
|
||||
def test_cpu_fp8_per_tensor_kernel_rejects_non_multiple_of_32_n(n: int):
|
||||
"""The AMX tinygemm kernel tiles N in chunks of 32 and cannot handle a
|
||||
remainder tile smaller than that, so can_implement must reject any N
|
||||
that isn't a multiple of 32 rather than let it crash the process.
|
||||
"""
|
||||
config = FP8ScaledMMLinearLayerConfig(
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_shape=(n, 64),
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
)
|
||||
supported, _ = CPUFp8PerTensorScaledMMLinearKernel.can_implement(config)
|
||||
assert not supported
|
||||
|
||||
|
||||
def reference_dynamic_fp8_linear(
|
||||
x: torch.Tensor, # BF16 [M, K]
|
||||
weight_fp8: torch.Tensor, # FP8 [N, K]
|
||||
weight_scale: torch.Tensor, # float32 [N, G]
|
||||
bias: torch.Tensor | None,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Dynamic per-token act x per-group weight: quantize x per-token,
|
||||
dequant weight per-group, matmul."""
|
||||
N, K = weight_fp8.shape
|
||||
G = weight_scale.shape[1]
|
||||
group_size = K // G
|
||||
x_q, x_scale = quantize_act_per_token(x)
|
||||
x_dq = x_q.float() * x_scale
|
||||
w_dq = (
|
||||
weight_fp8.float().view(N, G, group_size) * weight_scale.view(N, G, 1)
|
||||
).view(N, K)
|
||||
out = torch.mm(x_dq, w_dq.t())
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# W8A8 tests: fp8_scaled_mm_with_quant
|
||||
# ---------------------------------------------------------------------------
|
||||
W8A8_M_SIZES = [1, 4, 16, 64]
|
||||
W8A8_NK_SIZES = [(128, 256), (256, 512), (64, 128)]
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("static_activation", [True, False])
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_tensor(
|
||||
M: int, N: int, K: int, use_bias: bool, static_activation: bool
|
||||
):
|
||||
"""CPUFP8W8A8ScaledMMLinearKernel with per-tensor weight scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_tensor(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
|
||||
if static_activation:
|
||||
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_static_fp8_linear(
|
||||
x, w_fp8, w_scale, act_scale, bias, out_dtype
|
||||
)
|
||||
atol = 0.05
|
||||
else:
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
atol = 0.05
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("static_activation", [True, False])
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_channel(
|
||||
M: int, N: int, K: int, use_bias: bool, static_activation: bool
|
||||
):
|
||||
"""CPUFP8W8A8ScaledMMLinearKernel with per-channel weight scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_channel(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
|
||||
if static_activation:
|
||||
act_scale = (x.float().abs().max() / FP8_MAX).clamp(min=1e-7).view(1)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, act_scale, False, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_static_fp8_linear(
|
||||
x, w_fp8, w_scale, act_scale, bias, out_dtype
|
||||
)
|
||||
atol = 0.05
|
||||
else:
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
atol = 0.05
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=atol)
|
||||
|
||||
|
||||
@requires_cpu_fp8_w8a8
|
||||
@pytest.mark.parametrize("M", W8A8_M_SIZES)
|
||||
@pytest.mark.parametrize("N,K", W8A8_NK_SIZES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_cpu_fp8_w8a8_per_group(M: int, N: int, K: int, use_bias: bool):
|
||||
"""CPUFp8W8A8BlockScaledMMKernel: per-group (128-block) weight scale +
|
||||
dynamic per-token activation scale."""
|
||||
torch.manual_seed(42)
|
||||
out_dtype = torch.bfloat16
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
weight = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
bias = torch.randn(N, dtype=torch.float32) if use_bias else None
|
||||
|
||||
w_fp8, w_scale = quantize_weight_per_group(weight)
|
||||
packed_weight, packed_scale = torch.ops._C.float8_linear_prepack_cpu(w_fp8, w_scale)
|
||||
kernel_out = ops.fp8_scaled_mm_with_quant(
|
||||
x, None, True, packed_weight, packed_scale, bias, out_dtype
|
||||
)
|
||||
ref_out = reference_dynamic_fp8_linear(x, w_fp8, w_scale, bias, out_dtype)
|
||||
|
||||
assert kernel_out.dtype == out_dtype
|
||||
torch.testing.assert_close(kernel_out, ref_out, rtol=0.05, atol=0.05)
|
||||
@@ -1,771 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cutlass kernels.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`.
|
||||
"""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import baseline_scaled_mm, opcheck, to_fp8, to_int8
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("These tests use CUTLASS which requires CUDA", allow_module_level=True)
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 256, 128),
|
||||
(1, 16384, 1024),
|
||||
(1, 24576, 496),
|
||||
(16, 256, 496),
|
||||
(16, 16384, 128),
|
||||
(16, 24576, 4096),
|
||||
(32, 8192, 4096),
|
||||
(32, 16384, 4096),
|
||||
(33, 1024, 1024),
|
||||
(33, 8192, 128),
|
||||
(64, 2048, 496),
|
||||
(64, 16384, 1024),
|
||||
(100, 8192, 496),
|
||||
(128, 32768, 4096),
|
||||
(256, 4096, 4096),
|
||||
(512, 256, 1024),
|
||||
(512, 8192, 4096),
|
||||
(512, 16384, 128),
|
||||
(512, 24576, 128),
|
||||
]
|
||||
|
||||
# Shapes with N or K not divisible by 16. These exercise the padding path
|
||||
# inside CutlassFP8ScaledMMLinearKernel.apply_scaled_mm (e.g. Qwen2.5-VL
|
||||
# vision MLP dims).
|
||||
UNALIGNED_MNK_FACTORS = [
|
||||
(32, 3420, 1280),
|
||||
(32, 1280, 6840),
|
||||
(1, 3420, 1280),
|
||||
(64, 6840, 1280),
|
||||
(16, 100, 200),
|
||||
(33, 255, 513),
|
||||
]
|
||||
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
# -1 means full extent in that dimension
|
||||
TENSORWISE_GROUP_SHAPE = (-1, -1)
|
||||
PER_TOKEN_GROUP_SHAPE = (1, -1)
|
||||
PER_OUT_CH_GROUP_SHAPE = (-1, 1)
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
# Fall back to 0 so import doesn't fail on unknown capability
|
||||
capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0
|
||||
|
||||
|
||||
def rand_int8(shape: tuple, device: str = "cuda"):
|
||||
return to_int8(torch.rand(shape, device=device) * 255 - 128)
|
||||
|
||||
|
||||
def group_scale_helper(shape, group_shape):
|
||||
return [shape[i] if s < 0 else s for i, s in enumerate(group_shape)]
|
||||
|
||||
|
||||
def scale_shape(shape, group_shape):
|
||||
assert len(shape) == len(group_shape)
|
||||
group_shape = group_scale_helper(shape, group_shape)
|
||||
return tuple(cdiv(shape[i], group_shape[i]) for i in range(len(group_shape)))
|
||||
|
||||
|
||||
def cutlass_fp8_gemm_helper(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
a_scale_group_shape: tuple,
|
||||
b_scale_group_shape: tuple,
|
||||
use_bias: bool,
|
||||
out_dtype: type[torch.dtype] = torch.bfloat16,
|
||||
device: str = "cuda",
|
||||
):
|
||||
# Test for a cutlass kernel with per-token activation quantization
|
||||
# and per-output channel weight quantization.
|
||||
a = to_fp8(torch.randn((m, k), device=device))
|
||||
b = to_fp8(torch.randn((n, k), device=device).t())
|
||||
|
||||
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
|
||||
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
|
||||
|
||||
scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32)
|
||||
scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32)
|
||||
|
||||
# make scales M-major for blockwise quant, doesn't affect 1D scales
|
||||
scale_a = scale_a.t().contiguous().t()
|
||||
# make scales K-major for blockwise quant, doesn't affect 1D scales
|
||||
scale_b = scale_b.t().contiguous().t()
|
||||
|
||||
bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None
|
||||
|
||||
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1)
|
||||
|
||||
opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias))
|
||||
|
||||
|
||||
def cutlass_int8_gemm_helper(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
a_scale_group_shape: tuple,
|
||||
b_scale_group_shape: tuple,
|
||||
use_bias: bool,
|
||||
out_dtype: type[torch.dtype] = torch.bfloat16,
|
||||
device: str = "cuda",
|
||||
):
|
||||
# Test for a cutlass kernel with per-token activation quantization
|
||||
# and per-output channel weight quantization.
|
||||
a = to_int8(torch.randn((m, k), device=device) * 5)
|
||||
b = to_int8(torch.randn((n, k), device=device).t() * 5)
|
||||
|
||||
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
|
||||
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
|
||||
|
||||
scale_a = torch.randn(a_scales_shape, device=device, dtype=torch.float32)
|
||||
scale_b = torch.randn(b_scales_shape, device=device, dtype=torch.float32)
|
||||
|
||||
bias = torch.rand((n,), device=device, dtype=out_dtype) * 10 if use_bias else None
|
||||
|
||||
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
|
||||
|
||||
opcheck(torch.ops._C.cutlass_scaled_mm, (out, a, b, scale_a, scale_b, bias))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(89),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_gemm(
|
||||
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", UNALIGNED_MNK_FACTORS)
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(89),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_gemm_padded(
|
||||
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
"""Test CUTLASS FP8 GEMM with padding for non-16-aligned N/K dims.
|
||||
|
||||
Exercises CutlassFP8ScaledMMLinearKernel.apply_scaled_mm which pads
|
||||
inputs to satisfy CUTLASS alignment requirements — the path taken by
|
||||
models like Qwen2.5-VL whose vision MLP has non-16-aligned dims.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
)
|
||||
|
||||
a = to_fp8(torch.randn((m, k), device="cuda"))
|
||||
b = to_fp8(torch.randn((n, k), device="cuda").t())
|
||||
|
||||
a_scales_shape = scale_shape(a.shape, a_scale_group_shape)
|
||||
b_scales_shape = scale_shape(b.shape, b_scale_group_shape)
|
||||
|
||||
scale_a = torch.randn(a_scales_shape, device="cuda", dtype=torch.float32)
|
||||
scale_b = torch.randn(b_scales_shape, device="cuda", dtype=torch.float32)
|
||||
|
||||
scale_a = scale_a.t().contiguous().t()
|
||||
scale_b = scale_b.t().contiguous().t()
|
||||
|
||||
out_dtype = torch.bfloat16
|
||||
bias = torch.rand((n,), device="cuda", dtype=out_dtype) * 10 if use_bias else None
|
||||
|
||||
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
# process_weights_after_loading pad b to 16
|
||||
pad_k = (16 - k % 16) % 16
|
||||
pad_n = (16 - n % 16) % 16
|
||||
if pad_k > 0 or pad_n > 0:
|
||||
b = torch.nn.functional.pad(b.t().contiguous(), (0, pad_k, 0, pad_n)).t()
|
||||
if pad_n > 0 and scale_b.numel() > 1:
|
||||
scale_b = torch.nn.functional.pad(scale_b, (0, pad_n), value=1.0)
|
||||
|
||||
kernel = object.__new__(CutlassFP8ScaledMMLinearKernel)
|
||||
kernel.logical_output_size = n
|
||||
out = kernel.apply_scaled_mm(
|
||||
A=a,
|
||||
B=b,
|
||||
out_dtype=out_dtype,
|
||||
As=scale_a,
|
||||
Bs=scale_b,
|
||||
bias=bias,
|
||||
output_shape=[m, n],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, baseline, rtol=5e-1, atol=1.5e-1)
|
||||
|
||||
|
||||
# Two prefill-sized cases for the SM 12.x blockwise path, where the op switches
|
||||
# the tile scheduler to a swizzled CTA order once the weight exceeds the L2
|
||||
# (16384x2560 = 40 MiB and 5120x5120 = 25 MiB on a 24 MiB L2 part); the odd M
|
||||
# also covers the non-swap-AB dispatch. Elsewhere they run the default order
|
||||
# like the rest of the list.
|
||||
BLOCKWISE_PREFILL_FACTORS = [
|
||||
(8193, 16384, 2560),
|
||||
(5120, 5120, 5120),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS + BLOCKWISE_PREFILL_FACTORS)
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90),
|
||||
reason="FP8 blockwise is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_blockwise_scale_gemm(
|
||||
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
if k % b_scale_group_shape[0] != 0 or n % b_scale_group_shape[1] != 0:
|
||||
return
|
||||
if m % a_scale_group_shape[0] != 0 or k % a_scale_group_shape[1] != 0:
|
||||
return
|
||||
cutlass_fp8_gemm_helper(m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
def test_cutlass_int8_gemm(
|
||||
m: int, n: int, k: int, a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
cutlass_int8_gemm_helper(
|
||||
m, n, k, a_scale_group_shape, b_scale_group_shape, use_bias
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
def test_cutlass_int8_gemm_output_dtype(
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
out_dtype: type[torch.dtype],
|
||||
use_bias: bool,
|
||||
):
|
||||
cutlass_int8_gemm_helper(
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
use_bias,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(89),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_gemm_output_dtype(
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
out_dtype: type[torch.dtype],
|
||||
use_bias: bool,
|
||||
):
|
||||
cutlass_fp8_gemm_helper(
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
use_bias,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape,b_scale_group_shape", [((1, 128), (128, 128))]
|
||||
)
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_bias", [False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90),
|
||||
reason="FP8 blockwise is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_blockwise_scale_gemm_dtype(
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
out_dtype: type[torch.dtype],
|
||||
use_bias: bool,
|
||||
):
|
||||
cutlass_fp8_gemm_helper(
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
use_bias,
|
||||
out_dtype=out_dtype,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(89),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_gemm_devices(
|
||||
a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str
|
||||
):
|
||||
cutlass_fp8_gemm_helper(
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
use_bias,
|
||||
torch.bfloat16,
|
||||
device,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_cutlass_int8_gemm_devices(
|
||||
a_scale_group_shape, b_scale_group_shape, use_bias: bool, device: str
|
||||
):
|
||||
cutlass_int8_gemm_helper(
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
a_scale_group_shape,
|
||||
b_scale_group_shape,
|
||||
use_bias,
|
||||
out_dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
# For the following two tests:
|
||||
# N and K correspond to the size of the weight matrix and likely to be multiples
|
||||
# of a large power of two. In any case, the kernel will have a naive fallback
|
||||
# when N and K are not divisible by 16. But M is the number of tokens and the
|
||||
# kernel must handle any M thrown at it.
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(89),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_gemm_m_sweep(
|
||||
a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
for nk in range(32, 128, 32):
|
||||
for m in range(1, 128):
|
||||
cutlass_fp8_gemm_helper(
|
||||
m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"a_scale_group_shape", [PER_TOKEN_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"b_scale_group_shape", [PER_OUT_CH_GROUP_SHAPE, TENSORWISE_GROUP_SHAPE]
|
||||
)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
def test_cutlass_int8_gemm_m_sweep(
|
||||
a_scale_group_shape, b_scale_group_shape, use_bias: bool
|
||||
):
|
||||
for nk in range(32, 128, 32):
|
||||
for m in range(1, 128):
|
||||
cutlass_int8_gemm_helper(
|
||||
m, nk, nk, a_scale_group_shape, b_scale_group_shape, use_bias
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [32, 64, 128])
|
||||
@pytest.mark.parametrize("n", [16, 32, 64])
|
||||
@pytest.mark.parametrize("k", [64, 128, 256])
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skip
|
||||
def test_cutlass_int8_azp_bias_fold(m: int, n: int, k: int, out_dtype: torch.dtype):
|
||||
# Currently, the test is failing because folding azp into
|
||||
# 16-bit bias loses too much precision
|
||||
scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
|
||||
scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
|
||||
|
||||
aq_i8 = rand_int8((m, k))
|
||||
bq_i8 = rand_int8((n, k)).t()
|
||||
|
||||
aq_i32 = aq_i8.to(dtype=torch.int32)
|
||||
bq_i32 = bq_i8.to(dtype=torch.int32)
|
||||
|
||||
aq_f32 = aq_i8.to(dtype=torch.float32)
|
||||
bq_f32 = bq_i8.to(dtype=torch.float32)
|
||||
|
||||
b_dq = scale_b * bq_f32
|
||||
|
||||
azp_a = torch.rand((1,), device="cuda", dtype=torch.float32) * 10 + 1.5
|
||||
azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
|
||||
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
|
||||
|
||||
a_dq = scale_a * (aq_i32 + azp_aq_i8).to(dtype=torch.float32)
|
||||
torch.testing.assert_close(a_dq, scale_a * aq_f32 + azp_a)
|
||||
|
||||
baseline_dq = torch.mm(a_dq, b_dq).to(out_dtype)
|
||||
|
||||
J = torch.ones((1, k), device="cuda", dtype=torch.float32)
|
||||
azp_bias = (azp_a * scale_b * (J @ bq_f32)).to(out_dtype)
|
||||
assert azp_bias.shape == (1, n)
|
||||
assert azp_bias[0, :].shape == (n,)
|
||||
|
||||
baseline_q = (
|
||||
scale_a.to(device="cpu")
|
||||
* scale_b.to(device="cpu")
|
||||
* ((aq_i32 + azp_aq_i8).to(device="cpu") @ bq_i32.to(device="cpu"))
|
||||
).to(dtype=out_dtype, device="cuda")
|
||||
|
||||
out = ops.cutlass_scaled_mm(
|
||||
aq_i8, bq_i8, scale_a, scale_b, out_dtype=out_dtype, bias=azp_bias[0, :]
|
||||
)
|
||||
torch.testing.assert_close(out, baseline_dq, rtol=1e-2, atol=1e0)
|
||||
torch.testing.assert_close(out, baseline_q, rtol=1e-2, atol=1e0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [32, 64, 128])
|
||||
@pytest.mark.parametrize("n", [16, 32, 64])
|
||||
@pytest.mark.parametrize("k", [64, 128, 256])
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.parametrize("azp_per_token", [True, False])
|
||||
def test_cutlass_int8_azp(
|
||||
m: int, n: int, k: int, out_dtype: torch.dtype, use_bias: bool, azp_per_token: bool
|
||||
):
|
||||
m_azp = m if azp_per_token else 1
|
||||
scale_a = torch.randn((m_azp, 1), device="cuda", dtype=torch.float32) / 10
|
||||
scale_b = torch.randn((1, n), device="cuda", dtype=torch.float32) / 10
|
||||
|
||||
aq_i8 = rand_int8((m, k))
|
||||
aq_i32 = aq_i8.to(dtype=torch.int32)
|
||||
aq_f32 = aq_i8.to(dtype=torch.float32)
|
||||
|
||||
bq_i8 = rand_int8((n, k)).t()
|
||||
bq_i32 = bq_i8.to(dtype=torch.int32)
|
||||
bq_f32 = bq_i8.to(dtype=torch.float32)
|
||||
b_dq = scale_b * bq_f32
|
||||
|
||||
azp_a = torch.rand((m_azp, 1), device="cuda", dtype=torch.float32) * 10 + 1.5
|
||||
azp_aq_i8 = (azp_a / scale_a).to(dtype=torch.int8)
|
||||
azp_a = azp_aq_i8.to(dtype=torch.float32) * scale_a # correct for rounding
|
||||
|
||||
a_dq = scale_a * (aq_i32 - azp_aq_i8).to(dtype=torch.float32)
|
||||
torch.testing.assert_close(a_dq, scale_a * aq_f32 - azp_a, rtol=1e-4, atol=1e-3)
|
||||
|
||||
if use_bias:
|
||||
bias = torch.rand((1, n), device="cuda", dtype=out_dtype) * 10 + 2.5
|
||||
else:
|
||||
bias = torch.zeros((1, n), device="cuda", dtype=out_dtype)
|
||||
|
||||
baseline_dq = (torch.mm(a_dq, b_dq) + bias).to(out_dtype)
|
||||
|
||||
# int32 mm not supported on CUDA
|
||||
a_noazp_i32_cpu = (aq_i32 - azp_aq_i8).to(device="cpu")
|
||||
cq = (a_noazp_i32_cpu @ bq_i32.to(device="cpu")).to(device="cuda")
|
||||
baseline_q = (scale_a * scale_b * cq + bias).to(dtype=out_dtype)
|
||||
|
||||
# Hadamard is just the sum of the cols
|
||||
azp_adj_i32 = bq_i32.sum(dim=0, keepdim=True, dtype=torch.int32)
|
||||
azp_i32 = azp_aq_i8.to(dtype=torch.int32)
|
||||
func_bias = bias if use_bias else None
|
||||
|
||||
if azp_per_token:
|
||||
out = ops.cutlass_scaled_mm_azp(
|
||||
aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_adj_i32, azp_i32, func_bias
|
||||
)
|
||||
else:
|
||||
azp_with_adj_i32 = azp_i32 * azp_adj_i32
|
||||
out = ops.cutlass_scaled_mm_azp(
|
||||
aq_i8, bq_i8, scale_a, scale_b, out_dtype, azp_with_adj_i32, None, func_bias
|
||||
)
|
||||
|
||||
# bfloat16 precision is 7-bit mantissa -> 2^-8 ~ 0.4%
|
||||
# float16 precision is 10-bit mantissa -> 2^-11 ~ 0.05%
|
||||
rtol = 1e-2 if out_dtype == torch.bfloat16 else 1e-3
|
||||
atol = 1e-3
|
||||
torch.testing.assert_close(out, baseline_dq, rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(out, baseline_q, rtol=rtol, atol=atol)
|
||||
|
||||
if azp_per_token:
|
||||
opcheck(
|
||||
torch.ops._C.cutlass_scaled_mm_azp,
|
||||
(out, aq_i8, bq_i8, scale_a, scale_b, azp_adj_i32, azp_i32, func_bias),
|
||||
)
|
||||
else:
|
||||
opcheck(
|
||||
torch.ops._C.cutlass_scaled_mm_azp,
|
||||
(out, aq_i8, bq_i8, scale_a, scale_b, azp_with_adj_i32, None, func_bias),
|
||||
)
|
||||
|
||||
|
||||
# Test working with a subset of A and B
|
||||
def test_cutlass_subset():
|
||||
big_m, big_n, big_k = 1024, 1024, 1024
|
||||
m, n, k = 512, 512, 512
|
||||
|
||||
whole_a = to_int8(torch.randn((big_m, big_k), device="cuda") * 5)
|
||||
whole_b = to_int8(torch.randn((big_n, big_k), device="cuda").t() * 5)
|
||||
a = whole_a[0:m, 0:k]
|
||||
b = whole_b[0:k, 0:n]
|
||||
|
||||
scale_a = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
|
||||
scale_b = torch.randn((1, 1), device="cuda", dtype=torch.float32) / 10
|
||||
|
||||
out = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16)
|
||||
baseline = baseline_scaled_mm(a, b, scale_a, scale_b, out_dtype=torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
|
||||
|
||||
|
||||
# Test to make sure cuda graphs work
|
||||
class CutlassLayer(torch.nn.Module):
|
||||
def __init__(self, b, scale_a, scale_b, out_dtype):
|
||||
super().__init__()
|
||||
self.b = b
|
||||
self.scale_a = scale_a
|
||||
self.scale_b = scale_b
|
||||
self.out_dtype = out_dtype
|
||||
|
||||
def forward(self, a):
|
||||
return ops.cutlass_scaled_mm(
|
||||
a, self.b, self.scale_a, self.scale_b, self.out_dtype
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("per_act_token", [True, False])
|
||||
@pytest.mark.parametrize("per_out_ch", [True, False])
|
||||
def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
|
||||
m, n, k = 512, 512, 512
|
||||
|
||||
a = to_int8(torch.randn((m, k), device="cuda"))
|
||||
b = to_int8(torch.randn((n, k), device="cuda").t())
|
||||
|
||||
m_a_scales = m if per_act_token else 1
|
||||
n_b_scales = n if per_out_ch else 1
|
||||
|
||||
scale_a = torch.randn((m_a_scales, 1), device="cuda", dtype=torch.float32) / 10
|
||||
scale_b = torch.randn((1, n_b_scales), device="cuda", dtype=torch.float32) / 10
|
||||
|
||||
# Construct a trivial model with a single layer that calls a CUTLASS kernel
|
||||
model = CutlassLayer(b, scale_a, scale_b, torch.bfloat16)
|
||||
|
||||
# Run the model with a cuda graph
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = model(a)
|
||||
out.zero_()
|
||||
g.replay()
|
||||
|
||||
baseline = torch.mm(
|
||||
scale_a * a.to(dtype=torch.float32), scale_b * b.to(dtype=torch.float32)
|
||||
).to(torch.bfloat16)
|
||||
torch.testing.assert_close(out, baseline, rtol=1e-1, atol=1e0)
|
||||
|
||||
|
||||
def test_cutlass_support_opcheck():
|
||||
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [8, 64])
|
||||
@pytest.mark.parametrize("per_act_token", [True, False])
|
||||
@pytest.mark.parametrize("per_out_ch", [True, False])
|
||||
@pytest.mark.parametrize("use_bias", [False])
|
||||
@pytest.mark.skipif(
|
||||
(lambda x: x is None or not ops.cutlass_group_gemm_supported(x.to_int()))(
|
||||
current_platform.get_device_capability()
|
||||
),
|
||||
reason="Grouped gemm is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_fp8_group_gemm(
|
||||
num_experts: int, per_act_token: bool, per_out_ch: bool, use_bias: bool
|
||||
):
|
||||
# Device and dtype setup
|
||||
device = "cuda"
|
||||
out_dtype = torch.half
|
||||
|
||||
# Create separate A, B, C tensors for each group
|
||||
a_tensors = []
|
||||
b_tensors = []
|
||||
a_scales_tensors = []
|
||||
b_scales_tensors = []
|
||||
baseline_tensors = []
|
||||
|
||||
expert_offsets = torch.zeros((num_experts + 1), device=device, dtype=torch.int64)
|
||||
|
||||
problem_sizes = torch.zeros((num_experts, 3), device=device, dtype=torch.int32)
|
||||
|
||||
if not per_act_token:
|
||||
one_scale_a = torch.randn((1, 1), device=device, dtype=torch.float32)
|
||||
|
||||
alignment = 16 # 128 // 8
|
||||
# For variation, each group has dimensions
|
||||
n_g = alignment * random.randint(1, 64)
|
||||
k_g = alignment * random.randint(1, 64)
|
||||
for g in range(num_experts):
|
||||
m_g = alignment * random.randint(1, 64)
|
||||
|
||||
expert_offsets[g + 1] = expert_offsets[g] + m_g
|
||||
problem_sizes[g][0] = m_g
|
||||
problem_sizes[g][1] = n_g
|
||||
problem_sizes[g][2] = k_g
|
||||
|
||||
m_a_scales = m_g if per_act_token else 1
|
||||
n_b_scales = n_g if per_out_ch else 1
|
||||
|
||||
# Create group-specific A and B (FP8) and output (FP16/FP32)
|
||||
a_g = to_fp8(torch.randn((m_g, k_g), device=device))
|
||||
b_g = to_fp8(torch.randn((n_g, k_g), device=device).t())
|
||||
a_tensors.append(a_g)
|
||||
b_tensors.append(b_g)
|
||||
|
||||
# Set up A/B scales
|
||||
scale_b = torch.randn((1, n_b_scales), device=device, dtype=torch.float32)
|
||||
b_scales_tensors.append(scale_b)
|
||||
|
||||
if per_act_token:
|
||||
scale_a = torch.randn((m_a_scales, 1), device=device, dtype=torch.float32)
|
||||
a_scales_tensors.append(scale_a)
|
||||
else:
|
||||
scale_a = one_scale_a
|
||||
|
||||
# Compute baseline result for this group
|
||||
baseline_g = baseline_scaled_mm(a_g, b_g, scale_a, scale_b, out_dtype, None)
|
||||
baseline_tensors.append(baseline_g)
|
||||
|
||||
a_tensors_stacked = torch.empty(
|
||||
(expert_offsets[num_experts], k_g), device=device, dtype=torch.float8_e4m3fn
|
||||
)
|
||||
b_tensors_stacked = torch.empty(
|
||||
(num_experts, n_g, k_g), device=device, dtype=torch.float8_e4m3fn
|
||||
)
|
||||
|
||||
for g in range(num_experts):
|
||||
a_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = a_tensors[g]
|
||||
b_tensors_stacked[g] = b_tensors[g].t()
|
||||
b_tensors_stacked = b_tensors_stacked.transpose(1, 2)
|
||||
|
||||
if per_act_token:
|
||||
a_scales_tensors_stacked = torch.empty(
|
||||
(expert_offsets[num_experts], 1), device=device, dtype=torch.float32
|
||||
)
|
||||
for g in range(num_experts):
|
||||
a_scales_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]] = (
|
||||
a_scales_tensors[g]
|
||||
)
|
||||
else:
|
||||
a_scales_tensors_stacked = one_scale_a
|
||||
|
||||
b_scales_tensors_stacked = torch.empty(
|
||||
(num_experts, n_b_scales), device=device, dtype=torch.float32
|
||||
)
|
||||
for g in range(num_experts):
|
||||
b_scales_tensors_stacked[g] = b_scales_tensors[g]
|
||||
|
||||
out_tensors_stacked = torch.zeros(
|
||||
(expert_offsets[num_experts], n_g), device=device, dtype=out_dtype
|
||||
)
|
||||
|
||||
ab_strides = torch.full(
|
||||
(num_experts,), a_tensors_stacked.stride(0), device="cuda", dtype=torch.int64
|
||||
)
|
||||
c_strides = torch.full(
|
||||
(num_experts,), out_tensors_stacked.stride(0), device="cuda", dtype=torch.int64
|
||||
)
|
||||
|
||||
ops.cutlass_moe_mm(
|
||||
out_tensors_stacked,
|
||||
a_tensors_stacked,
|
||||
b_tensors_stacked,
|
||||
a_scales_tensors_stacked,
|
||||
b_scales_tensors_stacked,
|
||||
expert_offsets[:-1],
|
||||
problem_sizes,
|
||||
ab_strides,
|
||||
ab_strides,
|
||||
c_strides,
|
||||
per_act_token,
|
||||
per_out_ch,
|
||||
)
|
||||
|
||||
# Validate each group's result against the baseline
|
||||
for g in range(num_experts):
|
||||
baseline = baseline_tensors[g]
|
||||
c = out_tensors_stacked[expert_offsets[g] : expert_offsets[g + 1]]
|
||||
torch.testing.assert_close(c, baseline, rtol=1e-2, atol=5e-4)
|
||||
@@ -1,328 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the CUTLASS W4A8 kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_cutlass_w4a8.py`.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
convert_packed_uint4b8_to_signed_int4_inplace,
|
||||
pack_cols,
|
||||
pack_rows,
|
||||
quantize_weights,
|
||||
unpack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("These tests use CUTLASS which requires CUDA", allow_module_level=True)
|
||||
|
||||
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
|
||||
# unit tests to a common utility function. Currently the use of
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
# CUTLASS W4A8 kernels are only built for SM90 (9.0a); the kernel selector
|
||||
# requires exactly SM90 too, so match it here rather than accepting >= 90.
|
||||
IS_SUPPORTED_BY_GPU = (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability(90)
|
||||
)
|
||||
|
||||
MNK_SHAPES = [
|
||||
(1, 128, 128),
|
||||
(1, 512, 1024),
|
||||
(1, 4096, 4096),
|
||||
(1, 8192, 28672),
|
||||
(13, 8192, 4096),
|
||||
(26, 4096, 8192),
|
||||
(64, 4096, 4096),
|
||||
(64, 8192, 28672),
|
||||
(257, 128, 4096),
|
||||
(257, 4096, 4096),
|
||||
(1024, 4096, 8192),
|
||||
(1024, 8192, 4096),
|
||||
]
|
||||
|
||||
# TODO(czhu): get supported schedules from fn
|
||||
SCHEDULES = [
|
||||
"128x16_1x1x1",
|
||||
"256x16_1x1x1",
|
||||
"128x32_1x1x1",
|
||||
"256x32_1x1x1",
|
||||
"128x64_1x1x1",
|
||||
"256x64_1x1x1",
|
||||
"128x128_1x1x1",
|
||||
"256x128_1x1x1",
|
||||
"128x256_1x1x1",
|
||||
"128x256_2x1x1",
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TypeConfig:
|
||||
act_type: torch.dtype
|
||||
weight_type: ScalarType
|
||||
output_type: torch.dtype | None
|
||||
group_scale_type: torch.dtype | None
|
||||
channel_scale_type: torch.dtype | None
|
||||
token_scale_type: torch.dtype | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tensors:
|
||||
w_ref: torch.Tensor
|
||||
a_ref: torch.Tensor
|
||||
a: torch.Tensor
|
||||
w_q: torch.Tensor
|
||||
w_g_s: torch.Tensor
|
||||
w_ch_s: torch.Tensor
|
||||
w_tok_s: torch.Tensor
|
||||
|
||||
|
||||
# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
|
||||
# Ch Scales Type, Tok Scales Type)
|
||||
TestTypeTuple = tuple[
|
||||
list[torch.dtype], ScalarType, torch.dtype | None, torch.dtype | None, bool
|
||||
]
|
||||
TEST_TYPES = [
|
||||
*(
|
||||
TypeConfig(
|
||||
act_type=torch.float8_e4m3fn,
|
||||
weight_type=w_type,
|
||||
output_type=o_type,
|
||||
group_scale_type=torch.float8_e4m3fn,
|
||||
channel_scale_type=torch.float32,
|
||||
token_scale_type=torch.float32,
|
||||
)
|
||||
for w_type in [scalar_types.int4]
|
||||
# TODO(czhu): fp16 out type
|
||||
for o_type in [torch.bfloat16]
|
||||
),
|
||||
]
|
||||
|
||||
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
|
||||
# unit tests to a common utility function. Currently the use of
|
||||
|
||||
|
||||
# For testing quantized linear kernels
|
||||
def to_fp8(tensor: torch.Tensor):
|
||||
finfo = torch.finfo(torch.float8_e4m3fn)
|
||||
return tensor.clamp(min=finfo.min, max=finfo.max).to(dtype=torch.float8_e4m3fn)
|
||||
|
||||
|
||||
def cutlass_quantize_and_pack(
|
||||
atype: torch.dtype,
|
||||
w: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
stype: torch.dtype | None,
|
||||
group_size: int | None,
|
||||
zero_points: bool = False,
|
||||
):
|
||||
assert wtype.is_integer(), "TODO: support floating point weights"
|
||||
|
||||
w_ref, w_q, w_s, w_zp = quantize_weights(
|
||||
w, wtype, group_size=group_size, zero_points=zero_points
|
||||
)
|
||||
|
||||
# since scales are cast to fp8, we need to compute w_ref this way
|
||||
w_ref = (
|
||||
(w_q).to(torch.float32)
|
||||
* w_s.to(atype).to(torch.float32).repeat_interleave(group_size, dim=0)
|
||||
).to(atype)
|
||||
|
||||
# bit mask prevents sign extending int4 when packing
|
||||
w_q = pack_rows(w_q & 0x0F, wtype.size_bits, *w_q.shape)
|
||||
w_q = w_q.t().contiguous().t() # convert to col major
|
||||
|
||||
w_q_packed = ops.cutlass_encode_and_reorder_int4b(w_q)
|
||||
w_s_packed = ops.cutlass_pack_scale_fp8(w_s.to(atype))
|
||||
|
||||
return w_ref, w_q_packed, w_s_packed, w_zp
|
||||
|
||||
|
||||
def create_test_tensors(
|
||||
shape: tuple[int, int, int], types: TypeConfig, group_size: int | None
|
||||
) -> Tensors:
|
||||
m, n, k = shape
|
||||
|
||||
print(
|
||||
"create_test_tensors, shape:", shape, "types:", types, "group_size:", group_size
|
||||
)
|
||||
|
||||
a = to_fp8(torch.randn((m, k), device="cuda"))
|
||||
w = to_fp8(torch.randn((k, n), device="cuda"))
|
||||
|
||||
if types.group_scale_type is not None:
|
||||
w = w.to(types.group_scale_type)
|
||||
if w.dtype.itemsize == 1:
|
||||
w = w.to(torch.float16)
|
||||
|
||||
w_ref, w_q_packed, w_s, _ = cutlass_quantize_and_pack(
|
||||
a.dtype, w, types.weight_type, types.group_scale_type, group_size, False
|
||||
)
|
||||
|
||||
a_ref = a.to(torch.float32)
|
||||
w_ref = w_ref.to(torch.float32)
|
||||
|
||||
# for the practical use case we need per-tok scales for fp8 activations
|
||||
w_tok_s = torch.randn((m,), device="cuda", dtype=types.token_scale_type)
|
||||
w_ch_s = torch.randn((n,), device="cuda", dtype=types.channel_scale_type)
|
||||
|
||||
return Tensors(
|
||||
w_ref=w_ref,
|
||||
a_ref=a_ref,
|
||||
a=a,
|
||||
w_q=w_q_packed,
|
||||
w_g_s=w_s,
|
||||
w_ch_s=w_ch_s,
|
||||
w_tok_s=w_tok_s,
|
||||
)
|
||||
|
||||
|
||||
def mm_test_helper(
|
||||
types: TypeConfig,
|
||||
tensors: Tensors,
|
||||
group_size: int | None = None,
|
||||
schedule: str | None = None,
|
||||
):
|
||||
# CUTLASS upstream uses fp8 with fastaccum as reference
|
||||
# https://github.com/NVIDIA/cutlass/blob/main/examples/55_hopper_mixed_dtype_gemm/55_hopper_int4_fp8_gemm.cu#L406
|
||||
output_ref = torch._scaled_mm(
|
||||
tensors.a_ref.to(types.act_type),
|
||||
tensors.w_ref.to(types.act_type).t().contiguous().t(), # col major
|
||||
tensors.w_tok_s.unsqueeze(1),
|
||||
tensors.w_ch_s.unsqueeze(0),
|
||||
out_dtype=types.output_type,
|
||||
use_fast_accum=True,
|
||||
)
|
||||
|
||||
output = ops.cutlass_w4a8_mm(
|
||||
a=tensors.a,
|
||||
b_q=tensors.w_q,
|
||||
b_group_scales=tensors.w_g_s,
|
||||
b_group_size=group_size,
|
||||
b_channel_scales=tensors.w_ch_s,
|
||||
a_token_scales=tensors.w_tok_s,
|
||||
)
|
||||
|
||||
print(output)
|
||||
print(output_ref)
|
||||
|
||||
torch.testing.assert_close(
|
||||
output, output_ref.to(output.dtype), rtol=1e-2, atol=1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
|
||||
)
|
||||
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
|
||||
@pytest.mark.parametrize("types", TEST_TYPES)
|
||||
@pytest.mark.parametrize("schedule", SCHEDULES)
|
||||
def test_cutlass_w4a8(shape, types: TypeConfig, schedule):
|
||||
group_sizes = [128]
|
||||
for group_size in group_sizes:
|
||||
tensors = create_test_tensors(shape, types, group_size)
|
||||
mm_test_helper(types, tensors, group_size, schedule)
|
||||
|
||||
|
||||
# Test to make sure cuda graphs work
|
||||
class W4A8Layer(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.kwargs = kwargs
|
||||
|
||||
def forward(self, a):
|
||||
return ops.cutlass_w4a8_mm(a=a, **self.kwargs)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
|
||||
)
|
||||
def test_w4a8_cuda_graph():
|
||||
m, n, k = 512, 4096, 4096
|
||||
|
||||
a = to_fp8(torch.randn((m, k), device="cuda"))
|
||||
b = to_fp8(torch.randn((k, n), device="cuda"))
|
||||
|
||||
wtype = scalar_types.int4
|
||||
stype = torch.float8_e4m3fn
|
||||
group_size = 128
|
||||
zero_points = False
|
||||
|
||||
w_ref, w_q_packed, w_s, _ = cutlass_quantize_and_pack(
|
||||
a.dtype, b.to(torch.float16), wtype, stype, group_size, zero_points
|
||||
)
|
||||
|
||||
w_tok_s = torch.randn((m,), device="cuda", dtype=torch.float32)
|
||||
w_ch_s = torch.randn((n,), device="cuda", dtype=torch.float32)
|
||||
|
||||
# Construct a trivial model with a single layer that calls the kernel
|
||||
model = W4A8Layer(
|
||||
b_q=w_q_packed,
|
||||
b_group_scales=w_s,
|
||||
b_group_size=group_size,
|
||||
b_channel_scales=w_ch_s,
|
||||
a_token_scales=w_tok_s,
|
||||
)
|
||||
|
||||
output_ref = torch._scaled_mm(
|
||||
a,
|
||||
w_ref.to(a.dtype).t().contiguous().t(), # col major
|
||||
w_tok_s.unsqueeze(1),
|
||||
w_ch_s.unsqueeze(0),
|
||||
out_dtype=torch.bfloat16,
|
||||
use_fast_accum=True,
|
||||
)
|
||||
|
||||
# Run the model with a cuda graph
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
output = model(a)
|
||||
|
||||
output.zero_()
|
||||
g.replay()
|
||||
|
||||
torch.testing.assert_close(output, output_ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="CUTLASS W4A8 is not supported on this GPU type."
|
||||
)
|
||||
@pytest.mark.parametrize("shape", MNK_SHAPES)
|
||||
def test_convert_packed_uint4b8_to_signed_int4_inplace(shape):
|
||||
"""The W4A16 checkpoints encode the weights as int4b8 packed to int32.
|
||||
The CUTLASS kernels expect signed int4 packed to int32.
|
||||
This tests checks that the runtime int4b8 -> signed int4 conversion
|
||||
matches the offline conversion step exactly.
|
||||
"""
|
||||
_, N, K = shape
|
||||
# random weights packed to int32
|
||||
t = torch.randint(
|
||||
low=torch.iinfo(torch.int32).min,
|
||||
high=torch.iinfo(torch.int32).max + 1,
|
||||
size=(N, K // 8),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
# compute reference
|
||||
unpacked = unpack_quantized_values_into_int32(
|
||||
t.clone(), scalar_types.uint4b8, packed_dim=1
|
||||
)
|
||||
unpacked = unpacked - 8 # int4b8 -> signed int4
|
||||
ref = pack_cols(unpacked & 0x0F, 4, *unpacked.shape)
|
||||
|
||||
out = convert_packed_uint4b8_to_signed_int4_inplace(t.clone())
|
||||
|
||||
assert torch.equal(ref, out)
|
||||
assert not torch.equal(ref, t)
|
||||
@@ -1,337 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the CUTLASS-based W4A8 grouped GEMM kernel and the full MoE layer."""
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_rows,
|
||||
quantize_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
IS_SUPPORTED_BY_GPU = (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability(90)
|
||||
)
|
||||
|
||||
|
||||
def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
|
||||
finfo = torch.finfo(torch.float8_e4m3fn)
|
||||
return tensor.clamp(min=finfo.min, max=finfo.max).to(dtype=torch.float8_e4m3fn)
|
||||
|
||||
|
||||
def cutlass_quantize(
|
||||
atype: torch.dtype,
|
||||
w: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
stype: torch.dtype | None,
|
||||
group_size: int | None,
|
||||
zero_points: bool = False,
|
||||
):
|
||||
"""Quantize weights into W4 and compute reference dequantized weights.
|
||||
|
||||
Encoding/reordering of weights and packing of scales is deferred
|
||||
until after all experts are combined.
|
||||
"""
|
||||
assert wtype.is_integer(), "TODO: support floating point weights"
|
||||
|
||||
w_ref, w_q, w_s, w_zp = quantize_weights(
|
||||
w, wtype, group_size=group_size, zero_points=zero_points
|
||||
)
|
||||
|
||||
# Since scales are later cast to fp8, recompute w_ref in atype here.
|
||||
w_ref = (
|
||||
w_q.to(torch.float32)
|
||||
* w_s.to(atype).to(torch.float32).repeat_interleave(group_size, dim=0)
|
||||
).to(atype)
|
||||
|
||||
# Bit mask prevents sign extension of int4 when packing.
|
||||
w_q = pack_rows(w_q & 0x0F, wtype.size_bits, *w_q.shape)
|
||||
# Make weights row-major (N, K).
|
||||
w_q = w_q.t().contiguous()
|
||||
|
||||
return w_ref, w_q, w_s.to(atype), w_zp
|
||||
|
||||
|
||||
def cutlass_preprocess(
|
||||
w_q_experts: list[torch.Tensor], w_s_experts: list[torch.Tensor]
|
||||
):
|
||||
"""Reorder/encode expert weights and pack scales.
|
||||
|
||||
Returns:
|
||||
w_q_packed: Packed/encoded int4 weights for all experts.
|
||||
w_s_packed: Packed fp8 scales for all experts.
|
||||
packed_layout: Layout/stride metadata for grouped GEMM.
|
||||
|
||||
"""
|
||||
w_s_packed = ops.cutlass_pack_scale_fp8(torch.stack(w_s_experts))
|
||||
w_q_packed, packed_layout = ops.cutlass_encode_and_reorder_int4b_grouped(
|
||||
torch.stack(w_q_experts)
|
||||
) # expects dim 3
|
||||
return w_q_packed, w_s_packed, packed_layout
|
||||
|
||||
|
||||
GROUP_SIZE = 128
|
||||
# (num_experts, N, K)
|
||||
TEST_SHAPES = [
|
||||
(8, 512, 2048),
|
||||
(8, 2048, 2048),
|
||||
(64, 512, 1024),
|
||||
(64, 2048, 2048),
|
||||
(4, 2048, 768),
|
||||
(8, 768, 2048),
|
||||
(64, 1536, 2048),
|
||||
(128, 8192, 4096), # test overflow int32
|
||||
]
|
||||
ALIGNMENT = 16 # torch._scaled_mm alignment for M, needed for reference check
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoETestSetup:
|
||||
num_experts: int
|
||||
K: int
|
||||
N: int
|
||||
Ms: list[int]
|
||||
M_full: int
|
||||
a: torch.Tensor
|
||||
a_ref: torch.Tensor
|
||||
a_strides: torch.Tensor
|
||||
out: torch.Tensor
|
||||
c_strides: torch.Tensor
|
||||
per_tok_scales: torch.Tensor
|
||||
per_chan_scales: torch.Tensor
|
||||
w_refs: list[torch.Tensor]
|
||||
w_q_packed: torch.Tensor
|
||||
w_s_packed: torch.Tensor
|
||||
problem_sizes: torch.Tensor
|
||||
expert_offsets: torch.Tensor
|
||||
b_strides: torch.Tensor
|
||||
group_scale_strides: torch.Tensor
|
||||
|
||||
|
||||
def make_moe_test_setup(
|
||||
num_experts: int,
|
||||
K: int,
|
||||
N: int,
|
||||
*,
|
||||
alignment: int = ALIGNMENT,
|
||||
max_blocks: int = 64,
|
||||
device: str = "cuda",
|
||||
random_zero: bool = False,
|
||||
) -> MoETestSetup:
|
||||
"""Create a full set of tensors for testing cutlass_w4a8_moe_mm."""
|
||||
assert K % GROUP_SIZE == 0
|
||||
# Token counts per expert (multiples of `alignment`).
|
||||
Ms = [alignment * random.randint(1, max_blocks) for _ in range(num_experts)]
|
||||
|
||||
# set random experts to 0 tokens
|
||||
if random_zero and num_experts > 1:
|
||||
num_zero = max(1, num_experts // 8)
|
||||
zero_indices = random.sample(range(num_experts), k=num_zero)
|
||||
for idx in zero_indices:
|
||||
Ms[idx] = 0
|
||||
|
||||
M_full = sum(Ms)
|
||||
assert M_full > 0
|
||||
|
||||
# Activations.
|
||||
a = to_fp8(torch.randn((M_full, K), device=device))
|
||||
a_ref = a.to(torch.float32)
|
||||
a_strides = torch.full((num_experts,), K, dtype=torch.int64, device=device)
|
||||
|
||||
# Output buffer.
|
||||
out = torch.empty((M_full, N), dtype=torch.bfloat16, device=device)
|
||||
c_strides = torch.full((num_experts,), N, dtype=torch.int64, device=device)
|
||||
|
||||
# Channel/token scales.
|
||||
per_tok_scales = torch.randn((M_full, 1), dtype=torch.float32, device=device)
|
||||
per_chan_scales = torch.randn(
|
||||
(num_experts, N, 1), dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
# Expert weights and scales.
|
||||
wtype = scalar_types.int4
|
||||
atype = stype = torch.float8_e4m3fn
|
||||
w_refs, w_qs, w_ss = [], [], []
|
||||
for _ in range(num_experts):
|
||||
b = to_fp8(torch.randn((K, N), device=device))
|
||||
w_ref, w_q, w_s, _ = cutlass_quantize(
|
||||
atype, b.to(torch.float16), wtype, stype, GROUP_SIZE, zero_points=False
|
||||
)
|
||||
w_refs.append(w_ref)
|
||||
w_qs.append(w_q)
|
||||
w_ss.append(w_s)
|
||||
|
||||
w_q_packed, w_s_packed, packed_layout = cutlass_preprocess(w_qs, w_ss)
|
||||
|
||||
problem_sizes = torch.tensor(
|
||||
[[N, M, K] for M in Ms], dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
expert_offsets = torch.cat(
|
||||
[
|
||||
torch.tensor([0], dtype=torch.int64),
|
||||
torch.cumsum(torch.tensor(Ms, dtype=torch.int64), dim=0)[:-1],
|
||||
]
|
||||
).to(device=device)
|
||||
|
||||
# B strides and group scale strides.
|
||||
b_strides = packed_layout
|
||||
group_scale_strides = torch.zeros(
|
||||
(num_experts, 2), dtype=torch.int64, device=device
|
||||
)
|
||||
group_scale_strides[:, 0] = N
|
||||
|
||||
return MoETestSetup(
|
||||
num_experts=num_experts,
|
||||
K=K,
|
||||
N=N,
|
||||
Ms=Ms,
|
||||
M_full=M_full,
|
||||
a=a,
|
||||
a_ref=a_ref,
|
||||
a_strides=a_strides,
|
||||
out=out,
|
||||
c_strides=c_strides,
|
||||
per_tok_scales=per_tok_scales,
|
||||
per_chan_scales=per_chan_scales,
|
||||
w_refs=w_refs,
|
||||
w_q_packed=w_q_packed,
|
||||
w_s_packed=w_s_packed,
|
||||
problem_sizes=problem_sizes,
|
||||
expert_offsets=expert_offsets,
|
||||
b_strides=b_strides,
|
||||
group_scale_strides=group_scale_strides,
|
||||
)
|
||||
|
||||
|
||||
def compute_moe_reference_output(setup: MoETestSetup) -> torch.Tensor:
|
||||
"""Compute reference output using torch._scaled_mm per expert."""
|
||||
out_ref = torch.empty_like(setup.out)
|
||||
|
||||
ends = torch.cumsum(torch.tensor(setup.Ms), 0).tolist()
|
||||
starts = setup.expert_offsets.cpu().tolist()
|
||||
|
||||
for i in range(setup.num_experts):
|
||||
start, end = starts[i], ends[i]
|
||||
if start == end:
|
||||
continue
|
||||
|
||||
out_ref_i = torch._scaled_mm(
|
||||
setup.a_ref[start:end].to(torch.float8_e4m3fn),
|
||||
setup.w_refs[i].to(torch.float8_e4m3fn).t().contiguous().t(),
|
||||
setup.per_tok_scales[start:end], # (M, 1)
|
||||
setup.per_chan_scales[i].reshape(1, -1), # (1, N)
|
||||
out_dtype=torch.bfloat16,
|
||||
use_fast_accum=True,
|
||||
)
|
||||
out_ref[start:end] = out_ref_i
|
||||
|
||||
return out_ref
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU,
|
||||
reason="W4A8 Grouped GEMM is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", TEST_SHAPES)
|
||||
@pytest.mark.parametrize("random_zero", [True, False])
|
||||
def test_cutlass_w4a8_moe_mm_end_to_end(shape, random_zero):
|
||||
num_experts, N, K = shape
|
||||
set_random_seed(42)
|
||||
setup = make_moe_test_setup(
|
||||
num_experts=num_experts, K=K, N=N, max_blocks=64, random_zero=random_zero
|
||||
)
|
||||
|
||||
ops.cutlass_w4a8_moe_mm(
|
||||
setup.out,
|
||||
setup.a,
|
||||
setup.w_q_packed,
|
||||
setup.per_tok_scales,
|
||||
setup.per_chan_scales,
|
||||
setup.w_s_packed,
|
||||
GROUP_SIZE,
|
||||
setup.expert_offsets,
|
||||
setup.problem_sizes,
|
||||
setup.a_strides,
|
||||
setup.b_strides,
|
||||
setup.c_strides,
|
||||
setup.group_scale_strides,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
out_ref = compute_moe_reference_output(setup)
|
||||
torch.testing.assert_close(setup.out, out_ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
class W4A8MoELayer(torch.nn.Module):
|
||||
"""Minimal wrapper module to test cuda graphs."""
|
||||
|
||||
def __init__(self, setup: MoETestSetup):
|
||||
super().__init__()
|
||||
self.setup = setup
|
||||
|
||||
def forward(self, a: torch.Tensor) -> torch.Tensor:
|
||||
s = self.setup
|
||||
ops.cutlass_w4a8_moe_mm(
|
||||
s.out,
|
||||
a,
|
||||
s.w_q_packed,
|
||||
s.per_tok_scales,
|
||||
s.per_chan_scales,
|
||||
s.w_s_packed,
|
||||
GROUP_SIZE,
|
||||
s.expert_offsets,
|
||||
s.problem_sizes,
|
||||
s.a_strides,
|
||||
s.b_strides,
|
||||
s.c_strides,
|
||||
s.group_scale_strides,
|
||||
)
|
||||
return s.out
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU,
|
||||
reason="W4A8 Grouped GEMM is not supported on this GPU type.",
|
||||
)
|
||||
def test_cutlass_w4a8_moe_mm_cuda_graph():
|
||||
set_random_seed(42)
|
||||
# Fixed config for CUDA graph test (single parameter point).
|
||||
num_experts = 8
|
||||
K = 512
|
||||
N = 2048
|
||||
|
||||
setup = make_moe_test_setup(
|
||||
num_experts=num_experts,
|
||||
K=K,
|
||||
N=N,
|
||||
max_blocks=32,
|
||||
)
|
||||
|
||||
# Construct model that calls the grouped GEMM kernel.
|
||||
model = W4A8MoELayer(setup)
|
||||
|
||||
# Build reference output once.
|
||||
out_ref = compute_moe_reference_output(setup)
|
||||
|
||||
# Capture and run the model in a CUDA graph.
|
||||
a_static = setup.a.clone() # static input tensor for graph replay
|
||||
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out_static = model(a_static)
|
||||
|
||||
out_static.zero_()
|
||||
g.replay()
|
||||
|
||||
torch.testing.assert_close(out_static, out_ref, rtol=1e-2, atol=1e-2)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferTrtllmMxfp8LinearKernel,
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import flashinfer as vllm_flashinfer
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if not (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and has_flashinfer()
|
||||
):
|
||||
pytest.skip(
|
||||
reason="FlashInfer TRTLLM MXFP8 requires an SM100-family GPU",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(weight: torch.Tensor) -> torch.nn.Module:
|
||||
from flashinfer import SfLayout, mxfp8_quantize
|
||||
|
||||
weight_mxfp8, weight_scale = mxfp8_quantize(
|
||||
weight,
|
||||
sf_swizzle_layout=SfLayout.layout_linear,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(weight_mxfp8, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
weight_scale.view(weight.shape[0], weight.shape[1] // 32),
|
||||
requires_grad=False,
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 130, 256), (7, 256, 512), (128, 130, 768)])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_linear_numerics(
|
||||
shape: tuple[int, int, int],
|
||||
) -> None:
|
||||
torch.manual_seed(0)
|
||||
m, n, k = shape
|
||||
x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(
|
||||
Mxfp8LinearLayerConfig(weight_shape=(n, k))
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
output = kernel.apply_weights(layer, x)
|
||||
reference = torch.mm(x, weight.t())
|
||||
similarity = F.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
)
|
||||
|
||||
assert output.shape == (m, n)
|
||||
assert output.is_contiguous()
|
||||
assert similarity.item() > 0.98
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_custom_ops() -> None:
|
||||
x = torch.randn((7, 512), dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn((256, 512), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(
|
||||
Mxfp8LinearLayerConfig(weight_shape=(256, 512))
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.flashinfer_mxfp8_quantize_8x4.default,
|
||||
(x,),
|
||||
)
|
||||
x_mxfp8, x_scale = vllm_flashinfer.flashinfer_mxfp8_quantize_8x4(x)
|
||||
# SchemaCheckMode compares inputs with allclose, which CUDA does not
|
||||
# implement for float8. The numerical tests above guard input mutation.
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.mm_mxfp8.default,
|
||||
(
|
||||
x_mxfp8,
|
||||
layer.weight.t(),
|
||||
x_scale,
|
||||
layer.weight_scale,
|
||||
torch.bfloat16,
|
||||
"trtllm",
|
||||
True,
|
||||
),
|
||||
test_utils=(
|
||||
"test_autograd_registration",
|
||||
"test_faketensor",
|
||||
"test_aot_dispatch_dynamic",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxfp8_linear_cuda_graph() -> None:
|
||||
torch.manual_seed(0)
|
||||
m, n, k = 7, 130, 512
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device="cuda")
|
||||
layer = _make_layer(weight)
|
||||
kernel = FlashInferTrtllmMxfp8LinearKernel(
|
||||
Mxfp8LinearLayerConfig(weight_shape=(n, k))
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
static_x = torch.randn((m, k), dtype=torch.bfloat16, device="cuda")
|
||||
kernel.apply_weights(layer, static_x)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_output = kernel.apply_weights(layer, static_x)
|
||||
|
||||
new_x = torch.randn_like(static_x)
|
||||
static_x.copy_(new_x)
|
||||
graph.replay()
|
||||
eager_output = kernel.apply_weights(layer, new_x)
|
||||
|
||||
torch.testing.assert_close(graph_output, eager_output, rtol=0, atol=0)
|
||||
@@ -1,280 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
convert_swizzled_to_linear,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import NvFp4LinearLayerConfig
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
has_flashinfer_b12x_gemm,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
reason="Nvfp4 Requires compute capability of 10 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
# m, n, k
|
||||
SHAPES = [
|
||||
(128, 128, 64),
|
||||
(128, 128, 128),
|
||||
(256, 128, 64),
|
||||
(128, 256, 128),
|
||||
(1, 128, 128),
|
||||
]
|
||||
PAD_SHAPES = [(150, 128, 64), (128, 128, 96), (2, 128, 64), (3, 128, 96)]
|
||||
SHAPES.extend(PAD_SHAPES)
|
||||
|
||||
SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_shape", [(17,), (1, 17), (2, 17)])
|
||||
@pytest.mark.parametrize(
|
||||
"kernel_cls",
|
||||
[FlashInferCutlassNvFp4LinearKernel, FlashInferCuteDslNvFp4LinearKernel],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_nvfp4_preserves_quantized_input_shape(
|
||||
token_shape: tuple[int, ...],
|
||||
kernel_cls: type[FlashInferCutlassNvFp4LinearKernel]
|
||||
| type[FlashInferCuteDslNvFp4LinearKernel],
|
||||
) -> None:
|
||||
"""Pre-quantized inputs must match the tensor path and retain token dimensions."""
|
||||
supported, reason = kernel_cls.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
set_random_seed(42)
|
||||
hidden_size, output_size = 128, 128
|
||||
x = torch.randn(*token_shape, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(output_size, hidden_size, dtype=x.dtype, device=x.device)
|
||||
input_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / x.abs().max()).float()
|
||||
weight_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).float()
|
||||
x_fp4, x_blockscale = ops.scaled_fp4_quant(x.reshape(-1, hidden_size), input_scale)
|
||||
weight_fp4, weight_blockscale = ops.scaled_fp4_quant(weight, weight_scale)
|
||||
quantized_input = QuantizedActivation(
|
||||
data=x_fp4.reshape(*token_shape, hidden_size // 2),
|
||||
scale=x_blockscale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=x.shape,
|
||||
quant_key=kNvfp4Dynamic,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.input_size_per_partition = hidden_size
|
||||
layer.output_size_per_partition = output_size
|
||||
layer.weight = weight_fp4
|
||||
layer.weight_scale = weight_blockscale
|
||||
layer.input_global_scale_inv = input_scale
|
||||
layer.alpha = (input_scale * weight_scale).reciprocal()
|
||||
kernel = kernel_cls(NvFp4LinearLayerConfig())
|
||||
|
||||
expected = kernel.apply_weights(layer, x.reshape(-1, hidden_size))
|
||||
actual = kernel.apply_weights(layer, quantized_input)
|
||||
|
||||
assert actual.shape == (*token_shape, output_size)
|
||||
assert actual.dtype == x.dtype
|
||||
torch.testing.assert_close(actual.reshape(expected.shape), expected)
|
||||
|
||||
|
||||
def get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_sf,
|
||||
b_sf,
|
||||
a_global_scale,
|
||||
b_global_scale,
|
||||
m,
|
||||
n,
|
||||
dtype,
|
||||
block_size,
|
||||
device,
|
||||
is_sf_128x4_layout,
|
||||
):
|
||||
_, m_k = a_fp4.shape
|
||||
_, n_k = b_fp4.shape
|
||||
assert m_k == n_k
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_sf,
|
||||
a_global_scale,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
block_size=block_size,
|
||||
is_sf_128x4_layout=is_sf_128x4_layout,
|
||||
)
|
||||
b_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size
|
||||
)
|
||||
return torch.matmul(a_in_dtype, b_in_dtype.t())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("backend", ["cute-dsl", "cutlass", "cudnn", "trtllm", "b12x"])
|
||||
@pytest.mark.parametrize("autotune", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_nvfp4_gemm(
|
||||
dtype: torch.dtype,
|
||||
shape: tuple[int, int, int],
|
||||
seed: int,
|
||||
device: str,
|
||||
backend: str,
|
||||
autotune: bool,
|
||||
) -> None:
|
||||
if "trtllm" in backend and dtype == torch.float16:
|
||||
pytest.skip("Only torch.bfloat16 is supported for TRTLLM FP4 GEMM operations")
|
||||
if backend == "cute-dsl" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("FlashInfer cutedsl backend is only supported on SM10x")
|
||||
if backend == "b12x" and not current_platform.has_device_capability(120):
|
||||
pytest.skip("b12x FP4 GEMM requires SM120+ (CC 12.0+)")
|
||||
if backend == "b12x" and not has_flashinfer_b12x_gemm():
|
||||
pytest.skip("b12x FP4 GEMM backend not available in installed FlashInfer")
|
||||
|
||||
set_random_seed(seed)
|
||||
m, n, packed_k = shape
|
||||
k = packed_k * 2
|
||||
block_size = 16
|
||||
a_dtype = torch.randn((m, k), dtype=dtype, device=device)
|
||||
b_dtype = torch.randn((n, k), dtype=dtype, device=device)
|
||||
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
b_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
alpha = 1.0 / (a_global_scale * b_global_scale)
|
||||
|
||||
# ops.scaled_fp4_quant returns swizzled scales, while weights
|
||||
# from checkpoints are in linear scales.
|
||||
# cutlass and b12x use swizzled scales directly; trtllm needs them unswizzled.
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(
|
||||
a_dtype, a_global_scale, is_sf_swizzled_layout=True, backend=backend
|
||||
)
|
||||
is_sf_128x4_layout = not (backend == "trtllm" and m <= 32)
|
||||
|
||||
b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(
|
||||
b_dtype, b_global_scale, is_sf_swizzled_layout=True
|
||||
)
|
||||
|
||||
# get_ref_results unswizzles the scales internally.
|
||||
expected_out = get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_scale_interleaved,
|
||||
b_scale_interleaved,
|
||||
a_global_scale,
|
||||
b_global_scale,
|
||||
m,
|
||||
n,
|
||||
dtype,
|
||||
block_size,
|
||||
device,
|
||||
is_sf_128x4_layout,
|
||||
)
|
||||
|
||||
import flashinfer
|
||||
|
||||
if "trtllm" in backend:
|
||||
epilogue_tile_m = 128
|
||||
b_fp4 = flashinfer.shuffle_matrix_a(b_fp4.view(torch.uint8), epilogue_tile_m)
|
||||
b_scale_interleaved = convert_swizzled_to_linear(
|
||||
b_scale_interleaved, n, k, block_size
|
||||
)
|
||||
b_scale_interleaved = (
|
||||
flashinfer.shuffle_matrix_sf_a(
|
||||
b_scale_interleaved.view(torch.uint8), epilogue_tile_m
|
||||
)
|
||||
.reshape(b_scale_interleaved.shape)
|
||||
.view(torch.float8_e4m3fn)
|
||||
)
|
||||
|
||||
with flashinfer.autotune(autotune):
|
||||
out = flashinfer_scaled_fp4_mm(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_scale_interleaved,
|
||||
b_scale_interleaved,
|
||||
alpha,
|
||||
dtype,
|
||||
backend=backend,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_nvfp4_w4a16_linear(
|
||||
shape: tuple[int, int, int],
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
supported, reason = FlashInferCuteDslNvFp4W4A16LinearKernel.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
set_random_seed(seed)
|
||||
m, n, packed_k = shape
|
||||
k = packed_k * 2
|
||||
x = torch.randn((m, k), dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn((n, k), dtype=torch.bfloat16, device=device)
|
||||
weight_quant_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).to(
|
||||
torch.float32
|
||||
)
|
||||
weight_global_scale = weight_quant_scale.reciprocal()
|
||||
weight_fp4, weight_scale = ops.scaled_fp4_quant(
|
||||
weight,
|
||||
weight_quant_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
weight_ref = dequantize_to_dtype(
|
||||
weight_fp4,
|
||||
weight_scale,
|
||||
weight_global_scale,
|
||||
dtype=torch.bfloat16,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = n
|
||||
layer.weight = torch.nn.Parameter(weight_fp4, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
weight_global_scale, requires_grad=False
|
||||
)
|
||||
kernel = FlashInferCuteDslNvFp4W4A16LinearKernel(NvFp4LinearLayerConfig())
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
output = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = F.linear(x, weight_ref)
|
||||
assert output.shape == (m, n)
|
||||
torch.testing.assert_close(output, expected, atol=1e-1, rtol=1e-1)
|
||||
@@ -1,73 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import flashinfer_scaled_fp8_mm
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
reason="Flashinfer FP8 gemms requires compute capability of 10.0 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
# m, n, k
|
||||
SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)]
|
||||
PAD_SHAPES = [(150, 128, 64), (128, 128, 96)]
|
||||
SHAPES.extend(PAD_SHAPES)
|
||||
|
||||
SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("autotune", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_fp8_gemm(
|
||||
dtype: torch.dtype,
|
||||
shape: tuple[int, int, int],
|
||||
use_bias: bool,
|
||||
seed: int,
|
||||
device: str,
|
||||
autotune: bool,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
m, n, k = shape
|
||||
a = torch.randn((m, k), dtype=dtype, device=device)
|
||||
b = torch.randn((n, k), dtype=dtype, device=device) / k
|
||||
|
||||
a_fp8, a_scale = ops.scaled_fp8_quant(a)
|
||||
b_fp8, b_scale = ops.scaled_fp8_quant(b)
|
||||
|
||||
expected_out = torch.mm(
|
||||
a_scale * a_fp8.to(dtype=torch.float32),
|
||||
b_scale * b_fp8.to(dtype=torch.float32).t(),
|
||||
).to(dtype=dtype)
|
||||
|
||||
if use_bias:
|
||||
bias = torch.randn((n,), dtype=dtype, device=device)
|
||||
expected_out = expected_out + bias
|
||||
else:
|
||||
bias = None
|
||||
|
||||
import flashinfer
|
||||
|
||||
with flashinfer.autotune(autotune):
|
||||
out = flashinfer_scaled_fp8_mm(
|
||||
a_fp8,
|
||||
b_fp8.t(),
|
||||
a_scale,
|
||||
b_scale,
|
||||
dtype,
|
||||
bias=bias,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, expected_out, atol=1e-2, rtol=1e-2)
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import (
|
||||
FP8_DTYPE,
|
||||
ref_dynamic_per_tensor_fp8_quant,
|
||||
ref_dynamic_per_token_quant,
|
||||
)
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
HIDDEN_SIZES = [17, 1024, 1025, 1026, 5137, 8193]
|
||||
NUM_TOKENS = [1, 7, 4096]
|
||||
SCALE_UBS = [True, False]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
def opcheck_fp8_quant(
|
||||
output,
|
||||
input,
|
||||
scale=None,
|
||||
scale_ub=None,
|
||||
use_per_token_if_dynamic=False,
|
||||
group_shape=None,
|
||||
):
|
||||
if scale is not None:
|
||||
opcheck(
|
||||
torch.ops._C.static_scaled_fp8_quant,
|
||||
(output, input, scale, group_shape),
|
||||
)
|
||||
elif use_per_token_if_dynamic:
|
||||
scale = torch.empty(
|
||||
(input.shape[0], 1), device=input.device, dtype=torch.float32
|
||||
)
|
||||
opcheck(
|
||||
torch.ops._C.dynamic_per_token_scaled_fp8_quant,
|
||||
(output, input, scale, scale_ub),
|
||||
)
|
||||
else:
|
||||
scale = torch.empty(
|
||||
(input.numel() // input.shape[-1], 1),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
opcheck(torch.ops._C.dynamic_scaled_fp8_quant, (output, input, scale))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("do_scale_ub", SCALE_UBS)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_dynamic_per_token_fp8_quant(
|
||||
num_tokens: int, hidden_size: int, dtype: torch.dtype, do_scale_ub: bool, seed: int
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
|
||||
x = (
|
||||
torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") + 1e-6
|
||||
) # avoid nans
|
||||
|
||||
scale_ub = (
|
||||
torch.mean(x).to(dtype=torch.float32, device="cuda") if do_scale_ub else None
|
||||
)
|
||||
ref_out, ref_scales = ref_dynamic_per_token_quant(x, FP8_DTYPE, scale_ub)
|
||||
ops_out, ops_scales = ops.scaled_fp8_quant(
|
||||
x, scale_ub=scale_ub, use_per_token_if_dynamic=True
|
||||
)
|
||||
|
||||
torch.testing.assert_close(ref_scales, ops_scales)
|
||||
torch.testing.assert_close(
|
||||
ref_out.to(dtype=torch.float32), ops_out.to(dtype=torch.float32)
|
||||
)
|
||||
|
||||
opcheck_fp8_quant(ops_out, x, None, scale_ub, use_per_token_if_dynamic=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_dynamic_per_tensor_fp8_quant(
|
||||
num_tokens: int, hidden_size: int, dtype: torch.dtype, seed: int
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
|
||||
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
|
||||
ops_out, ops_scale = ops.scaled_fp8_quant(x)
|
||||
|
||||
torch.testing.assert_close(ref_scale, ops_scale)
|
||||
torch.testing.assert_close(
|
||||
ref_out.to(dtype=torch.float32), ops_out.to(dtype=torch.float32)
|
||||
)
|
||||
|
||||
opcheck_fp8_quant(ops_out, x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pad", [8, 16])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_dynamic_per_tensor_fp8_quant_strided_input(
|
||||
pad: int, dtype: torch.dtype, seed: int
|
||||
) -> None:
|
||||
"""Non-contiguous rows take the strided abs-max kernel; contiguous rows take
|
||||
the flat one. Both must produce exactly the same scale and bytes.
|
||||
|
||||
pad=8 (16 B for bf16) leaves rows misaligned for 32 B vector loads, pad=16
|
||||
keeps them aligned but non-contiguous.
|
||||
"""
|
||||
set_random_seed(seed)
|
||||
num_tokens, hidden_size = 64, 4096
|
||||
|
||||
full = torch.rand(num_tokens, hidden_size + pad, dtype=dtype, device="cuda")
|
||||
x = full[:, :hidden_size]
|
||||
assert not x.is_contiguous()
|
||||
|
||||
ops_out, ops_scale = ops.scaled_fp8_quant(x)
|
||||
flat_out, flat_scale = ops.scaled_fp8_quant(x.contiguous())
|
||||
assert torch.equal(ops_scale, flat_scale)
|
||||
assert torch.equal(ops_out.view(torch.uint8), flat_out.view(torch.uint8))
|
||||
|
||||
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
|
||||
assert torch.equal(ref_scale, ops_scale)
|
||||
assert torch.equal(ref_out.view(torch.uint8), ops_out.view(torch.uint8))
|
||||
|
||||
|
||||
# Regression test for a case with large activations where an int32 index cannot
|
||||
# represent the number of elements.
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
def test_fp8_quant_large(seed: int) -> None:
|
||||
set_random_seed(seed)
|
||||
|
||||
num_tokens = 1024000 # Mistral-Nemo's max_position_embeddings
|
||||
hidden_size = 1152 # Smallest hidden_size to reproduce the error
|
||||
dtype = torch.bfloat16
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
ref_out, scale = ref_dynamic_per_tensor_fp8_quant(x)
|
||||
ops_out, _ = ops.scaled_fp8_quant(x, scale)
|
||||
|
||||
# Minimize memory footprint in this test by freeing x and upconverting
|
||||
# the outputs in place. (torch.allclose does not support fp8)
|
||||
del x
|
||||
ref_out = ref_out.to(dtype=dtype)
|
||||
ops_out = ops_out.to(dtype=dtype)
|
||||
|
||||
torch.testing.assert_close(ref_out, ops_out)
|
||||
|
||||
|
||||
# Test static FP8 quantization with 2D group scales
|
||||
GROUP_SHAPES_2D = [
|
||||
(-1, -1), # Per-tensor
|
||||
(-1, 1), # Per-channel
|
||||
(1, -1), # Per-token
|
||||
(-1, 128), # Per-head quantization
|
||||
(1, 128), # DeepSeek-style per-token-per-group (group_m=1, group_n=128)
|
||||
(128, 128), # DeepSeek-style block quantization
|
||||
(1, 64), # Smaller group size
|
||||
(1, 16), # Small group (scalar path in kernel)
|
||||
(4, 256), # Non-trivial both dimensions
|
||||
]
|
||||
# Use sizes divisible by all group shapes
|
||||
NUM_TOKENS_GROUP = [128, 512]
|
||||
HIDDEN_SIZES_GROUP = [256, 1024, 2048]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_GROUP)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES_GROUP)
|
||||
@pytest.mark.parametrize("group_shape", GROUP_SHAPES_2D)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@torch.inference_mode()
|
||||
def test_static_fp8_quant_group_2d(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
group_shape: tuple[int, int],
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
) -> None:
|
||||
"""Test static FP8 quantization with 2D group scales using scaled_quantize."""
|
||||
# Normalize group_shape (-1 means full extent)
|
||||
norm_group_m = num_tokens if group_shape[0] == -1 else group_shape[0]
|
||||
norm_group_n = hidden_size if group_shape[1] == -1 else group_shape[1]
|
||||
|
||||
# Skip if sizes are not divisible by group shape
|
||||
if num_tokens % norm_group_m != 0 or hidden_size % norm_group_n != 0:
|
||||
pytest.skip(
|
||||
f"Skipping: ({num_tokens}, {hidden_size}) not divisible by "
|
||||
f"group_shape ({group_shape[0]}, {group_shape[1]})"
|
||||
)
|
||||
|
||||
set_random_seed(seed)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
ref_out, scale = scaled_quantize(
|
||||
x,
|
||||
GroupShape(*group_shape),
|
||||
current_platform.fp8_dtype(),
|
||||
compute_dtype=torch.float32,
|
||||
)
|
||||
ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape)
|
||||
|
||||
torch.testing.assert_close(scale, ops_scale)
|
||||
torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=1.2e-1, atol=1e-5)
|
||||
|
||||
opcheck_fp8_quant(ops_out, x, scale=scale)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS_GROUP)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES_GROUP)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("group_shape", [(1, -1), (-1, 1)]) # per-token, per-channel
|
||||
@torch.inference_mode()
|
||||
def test_static_fp8_quant_1d_scale(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
seed: int,
|
||||
group_shape: tuple[int, int],
|
||||
) -> None:
|
||||
"""Test static FP8 quantization with 1D scale (per-token or per-channel)."""
|
||||
set_random_seed(seed)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
ref_out, scale_2d = scaled_quantize(
|
||||
x, GroupShape(*group_shape), FP8_DTYPE, compute_dtype=torch.float32
|
||||
)
|
||||
|
||||
# Flatten scale to 1D for testing 1D scale path
|
||||
scale_1d = scale_2d.flatten()
|
||||
ops_out, ops_scale = ops.scaled_fp8_quant(
|
||||
x, scale=scale_1d, group_shape=group_shape
|
||||
)
|
||||
|
||||
torch.testing.assert_close(scale_1d, ops_scale)
|
||||
torch.testing.assert_close(ref_out.float(), ops_out.float(), rtol=0.12, atol=0.0)
|
||||
|
||||
opcheck_fp8_quant(ops_out, x, scale=scale_1d, group_shape=group_shape)
|
||||
@@ -1,29 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm import _custom_ops as ops # noqa: F401
|
||||
|
||||
|
||||
def test_gptq_shuffle_opcheck():
|
||||
weight = torch.randint(
|
||||
-2000000, 2000000, (1792, 4096), device="cuda", dtype=torch.int32
|
||||
)
|
||||
bit = 4
|
||||
opcheck(torch.ops._C.gptq_shuffle, (weight, bit))
|
||||
|
||||
|
||||
def test_gptq_gemm_opcheck():
|
||||
a = torch.rand((240, 4096), device="cuda", dtype=torch.float16)
|
||||
weight = torch.randint(
|
||||
-2000000, 2000000, (512, 6144), device="cuda", dtype=torch.int32
|
||||
)
|
||||
zeros = torch.zeros((32, 768), device="cuda", dtype=torch.int32)
|
||||
scales = torch.rand((32, 6144), device="cuda", dtype=torch.float16)
|
||||
use_exllama = True
|
||||
bit = 4
|
||||
# Test both GPTQv1 and GPTQv2 format
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, True, bit))
|
||||
opcheck(torch.ops._C.gptq_gemm, (a, weight, zeros, scales, use_exllama, False, bit))
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.transform import deterministic_hadamard_matrix
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"These tests require hadacore_transform, not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size", [1, 32])
|
||||
@pytest.mark.parametrize("hidden_dim", [2**n for n in range(10)])
|
||||
def test_hadacore(batch_size, hidden_dim, dtype=torch.bfloat16, device="cuda"):
|
||||
x = torch.eye(hidden_dim, dtype=dtype, device=device)
|
||||
hadamard = deterministic_hadamard_matrix(
|
||||
hidden_dim, dtype=torch.float64, device="cuda"
|
||||
) / math.sqrt(hidden_dim)
|
||||
|
||||
y = ops.hadacore_transform(x.clone())
|
||||
y_true = (x.to(hadamard.dtype) @ hadamard.T).to(y.dtype)
|
||||
assert torch.allclose(y, y_true)
|
||||
|
||||
y = ops.hadacore_transform(y)
|
||||
assert torch.allclose(y, x)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from https://github.com/sgl-project/sglang/blob/main/test/srt/test_int8_kernel.py
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quant_utils import native_per_token_group_quant_int8
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (current_platform.is_cuda_alike() or current_platform.is_xpu()):
|
||||
pytest.skip(
|
||||
"INT8 Triton kernels require a CUDA-alike or XPU device",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
if current_platform.is_cuda_alike() and not current_platform.has_device_capability(
|
||||
(7, 0)
|
||||
):
|
||||
pytest.skip("INT8 Triton requires CUDA 7.0 or higher", allow_module_level=True)
|
||||
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
assert A.shape[-1] == B.shape[-1], "Dimension mismatch"
|
||||
assert B.ndim == 2 and B.is_contiguous(), "B must be a 2D contiguous tensor"
|
||||
|
||||
# Reshape input
|
||||
M = A.numel() // A.shape[-1]
|
||||
B = B.t() # Transpose weight matrix
|
||||
N, K = B.shape
|
||||
origin_C_shape = A.shape[:-1] + (K,)
|
||||
A = A.reshape(M, N)
|
||||
|
||||
# As is per-token [M, 1], Bs is per-column [1, K]
|
||||
C = torch.matmul(A, B) # [M, K]
|
||||
C = As * C * Bs.view(1, -1) # Broadcast per-column scale
|
||||
|
||||
return C.reshape(origin_C_shape).to(output_dtype)
|
||||
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, topk, topk_weight, topk_ids):
|
||||
"""This function performs fused moe with per-column int8 quantization
|
||||
using native torch."""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = per_token_quant_int8(a)
|
||||
# Repeat tokens to match topk
|
||||
a_q = a_q.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
# Also repeat the scale
|
||||
a_s = a_s.view(B, -1, 1).repeat(1, topk, 1).reshape(-1, 1) # [B*topk, 1]
|
||||
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
|
||||
# Calculate routing
|
||||
topk_weight = topk_weight.view(-1)
|
||||
topk_ids = topk_ids.view(-1)
|
||||
# Process each expert
|
||||
for i in range(w1.shape[0]):
|
||||
mask = topk_ids == i
|
||||
if mask.sum():
|
||||
# First MLP layer: note that a_s is now per-token
|
||||
inter_out = native_w8a8_per_token_matmul(
|
||||
a_q[mask], w1[i], a_s[mask], w1_s[i], output_dtype=a.dtype
|
||||
)
|
||||
# Activation function
|
||||
act_out = SiluAndMul().forward_native(inter_out)
|
||||
# Quantize activation output with per-token
|
||||
act_out_q, act_out_s = per_token_quant_int8(act_out)
|
||||
|
||||
# Second MLP layer
|
||||
out[mask] = native_w8a8_per_token_matmul(
|
||||
act_out_q, w2[i], act_out_s, w2_s[i], output_dtype=a.dtype
|
||||
)
|
||||
# Apply routing weights and sum
|
||||
return (
|
||||
out.view(B, -1, w2.shape[1]) * topk_weight.view(B, -1, 1).to(out.dtype)
|
||||
).sum(dim=1)
|
||||
|
||||
|
||||
DTYPES = [torch.half, torch.bfloat16]
|
||||
M = [1, 33]
|
||||
N = [128, 1024]
|
||||
K = [256, 4096]
|
||||
E = [8]
|
||||
TOP_KS = [2, 6]
|
||||
SEEDS = [0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M, N, K, E, topk, dtype, seed",
|
||||
itertools.product(M, N, K, E, TOP_KS, DTYPES, SEEDS),
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_fp8_fused_moe(default_vllm_config, M, N, K, E, topk, dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
device = current_platform.device_type
|
||||
# Initialize int8 quantization parameters
|
||||
factor_for_scale = 1e-2
|
||||
int8_max = 127
|
||||
int8_min = -128
|
||||
|
||||
# Input tensor
|
||||
# M * K
|
||||
a = torch.randn((M, K), dtype=dtype, device=device) / 10
|
||||
|
||||
# Generate int8 weights
|
||||
w1_fp32 = (
|
||||
torch.rand(
|
||||
(E, 2 * N, K),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
- 0.5
|
||||
) * 2
|
||||
w1 = (w1_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8)
|
||||
|
||||
w2_fp32 = (torch.rand((E, K, N), dtype=torch.float32, device=device) - 0.5) * 2
|
||||
w2 = (w2_fp32 * int8_max).clamp(min=int8_min, max=int8_max).to(torch.int8)
|
||||
|
||||
# Generate scale for each column (per-column quantization)
|
||||
w1_s = torch.rand(E, 2 * N, device=w1_fp32.device) * factor_for_scale
|
||||
w2_s = torch.rand(E, K, device=w2_fp32.device) * factor_for_scale
|
||||
score = torch.randn((M, E), dtype=dtype, device=device)
|
||||
score = torch.softmax(score, dim=-1, dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(score, topk)
|
||||
|
||||
ref_out = torch_w8a8_per_column_moe(
|
||||
a, w1, w2, w1_s, w2_s, topk, topk_weights, topk_ids
|
||||
)
|
||||
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
torch.int8,
|
||||
per_act_token_quant=True,
|
||||
block_shape=None,
|
||||
w1_scale=w1_s,
|
||||
w2_scale=w2_s,
|
||||
)
|
||||
|
||||
out = fused_experts(
|
||||
a,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
# Check results
|
||||
rel_diff = torch.mean(
|
||||
torch.abs(out.to(torch.float32) - ref_out.to(torch.float32))
|
||||
) / torch.mean(torch.abs(ref_out.to(torch.float32)))
|
||||
assert rel_diff < 0.05
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 64])
|
||||
@pytest.mark.parametrize("hidden_dim", [128, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode()
|
||||
def test_per_token_quant_int8(num_tokens, hidden_dim, dtype):
|
||||
"""Covers the round_int8 Triton kernel via per_token_quant_int8."""
|
||||
torch.manual_seed(0)
|
||||
device = current_platform.device_type
|
||||
x = torch.randn((num_tokens, hidden_dim), dtype=dtype, device=device) * 8
|
||||
|
||||
q, s = per_token_quant_int8(x)
|
||||
# Per-token int8 quant is per-token-group quant with group_size = hidden_dim.
|
||||
ref_q, ref_s = native_per_token_group_quant_int8(x, hidden_dim)
|
||||
|
||||
assert q.dtype == torch.int8
|
||||
assert q.shape == (num_tokens, hidden_dim)
|
||||
assert s.shape == (num_tokens, 1)
|
||||
assert torch.allclose(s.float(), ref_s.float(), atol=1e-3, rtol=1e-3)
|
||||
# Quantized values may differ by at most 1 ULP due to rounding.
|
||||
assert torch.all((q.float() - ref_q.float()).abs() <= 1)
|
||||
@@ -1,450 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the machete kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_machete_mm.py`.
|
||||
"""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass, fields
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.machete_utils import (
|
||||
query_machete_supported_group_sizes,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
pack_rows,
|
||||
quantize_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"These tests require machete_prepack_B, not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
CUDA_DEVICES = [
|
||||
f"cuda:{i}" for i in range(1 if torch.accelerator.device_count() == 1 else 2)
|
||||
]
|
||||
|
||||
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
|
||||
# unit tests to a common utility function. Currently the use of
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
|
||||
|
||||
MNK_SHAPES = [
|
||||
(1, 128, 128),
|
||||
(1, 8192, 28672),
|
||||
(13, 8192, 4096),
|
||||
(26, 4096, 8192),
|
||||
(64, 4096, 4096),
|
||||
(64, 8192, 28672),
|
||||
(257, 128, 4096),
|
||||
(257, 4224, 4160),
|
||||
(1024, 8192, 4096),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TypeConfig:
|
||||
act_type: torch.dtype
|
||||
weight_type: ScalarType
|
||||
output_type: torch.dtype | None
|
||||
group_scale_type: torch.dtype | None
|
||||
group_zero_type: torch.dtype | None
|
||||
channel_scale_type: torch.dtype | None
|
||||
token_scale_type: torch.dtype | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tensors:
|
||||
w_ref: torch.Tensor
|
||||
a_ref: torch.Tensor
|
||||
a: torch.Tensor
|
||||
w_q: torch.Tensor
|
||||
w_g_s: torch.Tensor | None
|
||||
w_g_zp: torch.Tensor | None
|
||||
w_ch_s: torch.Tensor | None
|
||||
w_tok_s: torch.Tensor | None
|
||||
|
||||
|
||||
# (Act Type, Weight Type, Output Type, Scale Type, ZeroPoints,
|
||||
# Ch Scales Type, Tok Scales Type)
|
||||
# NOTE: None "Scale Type" means the act type is floating point
|
||||
# None "Output Type" means the output type is the same as the act type
|
||||
TestTypeTuple = tuple[
|
||||
list[torch.dtype], ScalarType, torch.dtype | None, torch.dtype | None, bool
|
||||
]
|
||||
TEST_TYPES = [
|
||||
# GPTQ style
|
||||
*(
|
||||
TypeConfig(
|
||||
act_type=a_type,
|
||||
weight_type=w_type,
|
||||
output_type=None,
|
||||
group_scale_type=a_type,
|
||||
group_zero_type=None,
|
||||
channel_scale_type=None,
|
||||
token_scale_type=None,
|
||||
)
|
||||
for w_type in [scalar_types.uint4b8, scalar_types.uint8b128]
|
||||
for a_type in [torch.float16, torch.bfloat16]
|
||||
),
|
||||
# AWQ style
|
||||
*(
|
||||
TypeConfig(
|
||||
act_type=a_type,
|
||||
weight_type=w_type,
|
||||
output_type=None,
|
||||
group_scale_type=a_type,
|
||||
group_zero_type=a_type,
|
||||
channel_scale_type=None,
|
||||
token_scale_type=None,
|
||||
)
|
||||
for w_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
for a_type in [torch.float16, torch.bfloat16]
|
||||
),
|
||||
# # QQQ style
|
||||
# *(TypeConfig(act_type=torch.int8,
|
||||
# weight_type=scalar_types.uint4b8,
|
||||
# output_type=torch.float16,
|
||||
# group_scale_type=group_scale_type,
|
||||
# group_zero_type=None,
|
||||
# channel_scale_type=torch.float,
|
||||
# token_scale_type=torch.float)
|
||||
# for group_scale_type in [None, torch.float16]),
|
||||
# *(TypeConfig(act_type=torch.float8_e4m3fn,
|
||||
# weight_type=scalar_types.uint4b8,
|
||||
# output_type=torch.float16,
|
||||
# group_scale_type=group_scale_type,
|
||||
# group_zero_type=None,
|
||||
# channel_scale_type=torch.float,
|
||||
# token_scale_type=torch.float)
|
||||
# for group_scale_type in [None, torch.float16]),
|
||||
]
|
||||
|
||||
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
|
||||
# unit tests to a common utility function. Currently the use of
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
|
||||
|
||||
|
||||
def rand_data(shape, dtype=torch.float16, scale=1, offset=0):
|
||||
if dtype.is_floating_point:
|
||||
return (scale * torch.rand(shape, device="cuda") - offset).to(dtype)
|
||||
else:
|
||||
return torch.randint(-8, 7, shape, dtype=dtype, device="cuda")
|
||||
|
||||
|
||||
def maybe_convert_zeropoints(zps: torch.Tensor | None, s: torch.Tensor):
|
||||
return zps if zps is None else -1 * s * (zps.to(s.dtype))
|
||||
|
||||
|
||||
def group_size_valid(shape: tuple[int, int, int], group_size: int | None) -> bool:
|
||||
return group_size is None or group_size == -1 or shape[2] % group_size == 0
|
||||
|
||||
|
||||
def machete_quantize_and_pack(
|
||||
atype: torch.dtype,
|
||||
w: torch.Tensor,
|
||||
wtype: ScalarType,
|
||||
stype: torch.dtype | None,
|
||||
group_size: int | None,
|
||||
zero_points: bool = False,
|
||||
):
|
||||
assert wtype.is_integer(), "TODO: support floating point weights"
|
||||
|
||||
w_ref, w_q, w_s, w_zp = quantize_weights(
|
||||
w,
|
||||
wtype,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
# to match how the kernel applies zps
|
||||
ref_zero_points_after_scales=True,
|
||||
)
|
||||
|
||||
w_q = pack_rows(w_q, wtype.size_bits, *w_q.shape)
|
||||
w_q = w_q.t().contiguous().t() # convert to col major
|
||||
|
||||
w_q_machete = ops.machete_prepack_B(w_q, atype, wtype, stype)
|
||||
opcheck(torch.ops._C.machete_prepack_B, (w_q, atype, wtype.id, stype))
|
||||
|
||||
return w_ref, w_q_machete, w_s, w_zp
|
||||
|
||||
|
||||
def create_test_tensors(
|
||||
shape: tuple[int, int, int],
|
||||
types: TypeConfig,
|
||||
group_size: int | None,
|
||||
subset_stride_factor: int | None = None,
|
||||
) -> Tensors:
|
||||
m, n, k = shape
|
||||
factor = subset_stride_factor or 1
|
||||
|
||||
print(
|
||||
"create_test_tensors, shape:", shape, "types:", types, "group_size:", group_size
|
||||
)
|
||||
|
||||
a = rand_data((m * factor, k * factor), types.act_type, scale=3, offset=2)
|
||||
w = rand_data((k * factor, n * factor), types.act_type, scale=3, offset=1)
|
||||
|
||||
if factor > 1:
|
||||
a = a[0:m, 0:k]
|
||||
w = w[0:k, 0:n]
|
||||
|
||||
if types.group_scale_type is not None:
|
||||
w = w.to(types.group_scale_type)
|
||||
if w.dtype.itemsize == 1:
|
||||
w = w.to(torch.float16)
|
||||
|
||||
w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
|
||||
a.dtype,
|
||||
w,
|
||||
types.weight_type,
|
||||
types.group_scale_type,
|
||||
group_size,
|
||||
types.group_zero_type is not None,
|
||||
)
|
||||
|
||||
if not a.dtype.is_floating_point:
|
||||
aiinfo = torch.iinfo(a.dtype)
|
||||
w_ref = w_ref.round().clamp(aiinfo.min, aiinfo.max)
|
||||
|
||||
a_ref = a.to(torch.float32)
|
||||
w_ref = w_ref.to(torch.float32)
|
||||
|
||||
w_ch_s = (
|
||||
None
|
||||
if types.channel_scale_type is None
|
||||
else rand_data((n,), types.channel_scale_type)
|
||||
)
|
||||
w_tok_s = (
|
||||
None
|
||||
if types.token_scale_type is None
|
||||
else rand_data((m,), types.token_scale_type)
|
||||
)
|
||||
|
||||
return Tensors(
|
||||
w_ref=w_ref,
|
||||
a_ref=a_ref,
|
||||
a=a,
|
||||
w_q=w_q_packed,
|
||||
w_g_s=w_s,
|
||||
w_g_zp=maybe_convert_zeropoints(w_zp, w_s),
|
||||
w_ch_s=w_ch_s,
|
||||
w_tok_s=w_tok_s,
|
||||
)
|
||||
|
||||
|
||||
# None stype means scales use the same dtype as a
|
||||
def machete_mm_test_helper(
|
||||
types: TypeConfig,
|
||||
tensors: Tensors,
|
||||
group_size: int | None = None,
|
||||
schedule: str | None = None,
|
||||
):
|
||||
output_ref = torch.matmul(tensors.a_ref, tensors.w_ref)
|
||||
output_ref_type = output_ref.dtype
|
||||
|
||||
if tensors.w_ch_s is not None:
|
||||
output_ref = (
|
||||
output_ref.to(tensors.w_ch_s.dtype) * tensors.w_ch_s.unsqueeze(0)
|
||||
).to(output_ref_type)
|
||||
if tensors.w_tok_s is not None:
|
||||
output_ref = (
|
||||
output_ref.to(tensors.w_tok_s.dtype) * tensors.w_tok_s.unsqueeze(1)
|
||||
).to(output_ref_type)
|
||||
|
||||
output = ops.machete_mm(
|
||||
a=tensors.a,
|
||||
b_q=tensors.w_q,
|
||||
b_type=types.weight_type,
|
||||
b_group_scales=tensors.w_g_s,
|
||||
b_group_zeros=tensors.w_g_zp,
|
||||
b_group_size=group_size,
|
||||
b_channel_scales=tensors.w_ch_s,
|
||||
a_token_scales=tensors.w_tok_s,
|
||||
out_type=types.output_type,
|
||||
schedule=schedule,
|
||||
)
|
||||
|
||||
print(output)
|
||||
print(output_ref)
|
||||
|
||||
# Relax atol as our reduction dim becomes larger (more rounding error)
|
||||
# Relax atol when we have zeropoints since the way machete applies
|
||||
# zeropoints (after scales) causes noise around 0
|
||||
atol = (
|
||||
1
|
||||
if tensors.w_g_zp is not None
|
||||
else min(5e-2 * math.sqrt(tensors.a.shape[1]), 1)
|
||||
)
|
||||
rtol = 1e-1 if tensors.a.element_size() >= 2 else 2e-1
|
||||
torch.testing.assert_close(
|
||||
output, output_ref.to(output.dtype), rtol=rtol, atol=atol
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
|
||||
)
|
||||
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
|
||||
@pytest.mark.parametrize("types", TEST_TYPES)
|
||||
def test_machete_all_schedules(shape, types: TypeConfig):
|
||||
group_sizes: list[int | None] = []
|
||||
if types.group_scale_type is None:
|
||||
group_sizes = [None]
|
||||
else:
|
||||
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
|
||||
|
||||
for group_size in group_sizes:
|
||||
if not group_size_valid(shape, group_size):
|
||||
continue
|
||||
|
||||
tensors = create_test_tensors(shape, types, group_size)
|
||||
print(f"MNK = {shape}")
|
||||
for schedule in ops.machete_supported_schedules(
|
||||
types.act_type,
|
||||
types.weight_type,
|
||||
group_scales_type=types.group_scale_type,
|
||||
group_zeros_type=types.group_scale_type,
|
||||
out_type=types.output_type,
|
||||
):
|
||||
print(f"Testing schedule {schedule}")
|
||||
machete_mm_test_helper(types, tensors, group_size, schedule)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
|
||||
)
|
||||
@pytest.mark.parametrize("shape", MNK_SHAPES, ids=lambda x: "x".join(str(v) for v in x))
|
||||
@pytest.mark.parametrize("types", TEST_TYPES)
|
||||
def test_machete_heuristic(shape, types: TypeConfig):
|
||||
group_sizes: list[int | None] = []
|
||||
if types.group_scale_type is None:
|
||||
group_sizes = [None]
|
||||
else:
|
||||
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
|
||||
|
||||
for group_size in group_sizes:
|
||||
if not group_size_valid(shape, group_size):
|
||||
continue
|
||||
|
||||
tensors = create_test_tensors(shape, types, group_size)
|
||||
machete_mm_test_helper(types, tensors, group_size)
|
||||
|
||||
|
||||
# Test working on other devices
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
|
||||
)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
def test_machete_devices(device: str):
|
||||
group_size = 128
|
||||
|
||||
type_config = TypeConfig(
|
||||
act_type=torch.float16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
output_type=None,
|
||||
group_scale_type=torch.float16,
|
||||
group_zero_type=None,
|
||||
channel_scale_type=None,
|
||||
token_scale_type=None,
|
||||
)
|
||||
|
||||
tensors = create_test_tensors((512, 4096, 4096), type_config, group_size)
|
||||
|
||||
for field in fields(Tensors):
|
||||
tensor = getattr(tensors, field.name)
|
||||
if isinstance(tensor, torch.Tensor):
|
||||
setattr(tensors, field.name, tensor.to(device))
|
||||
|
||||
machete_mm_test_helper(type_config, tensors, group_size)
|
||||
|
||||
|
||||
# Test working with a subset of A and B
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
|
||||
)
|
||||
def test_machete_subset():
|
||||
group_size = 128
|
||||
|
||||
type_config = TypeConfig(
|
||||
act_type=torch.float16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
output_type=None,
|
||||
group_scale_type=torch.float16,
|
||||
group_zero_type=None,
|
||||
channel_scale_type=None,
|
||||
token_scale_type=None,
|
||||
)
|
||||
|
||||
tensors = create_test_tensors(
|
||||
(512, 4096, 4096), type_config, group_size, subset_stride_factor=2
|
||||
)
|
||||
machete_mm_test_helper(type_config, tensors, group_size)
|
||||
|
||||
|
||||
# Test to make sure cuda graphs work
|
||||
class MacheteLayer(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
self.kwargs = kwargs
|
||||
|
||||
def forward(self, a):
|
||||
return ops.machete_mm(a=a, **self.kwargs)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not IS_SUPPORTED_BY_GPU, reason="Machete is not supported on this GPU type."
|
||||
)
|
||||
def test_machete_cuda_graph():
|
||||
m, n, k = 512, 4096, 4096
|
||||
|
||||
a = rand_data((m, k), torch.float16)
|
||||
b = rand_data((k, n), torch.float16)
|
||||
wtype = scalar_types.uint4b8
|
||||
stype = torch.float16
|
||||
group_size = 128
|
||||
zero_points = False
|
||||
|
||||
w_ref, w_q_packed, w_s, w_zp = machete_quantize_and_pack(
|
||||
a.dtype, b, wtype, stype, group_size, zero_points
|
||||
)
|
||||
|
||||
# Construct a trivial model with a single layer that calls a machete kernel
|
||||
model = MacheteLayer(
|
||||
b_q=w_q_packed,
|
||||
b_type=wtype,
|
||||
b_group_scales=w_s,
|
||||
b_group_zeros=maybe_convert_zeropoints(w_zp, w_s),
|
||||
b_group_size=group_size,
|
||||
)
|
||||
|
||||
output_ref = torch.matmul(a, w_ref)
|
||||
|
||||
# Run the model with a cuda graph
|
||||
stream = torch.cuda.Stream()
|
||||
with torch.cuda.stream(stream):
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
output = model(a)
|
||||
output.zero_()
|
||||
g.replay()
|
||||
|
||||
# Relax atol as our reduction dim becomes larger (more rounding error)
|
||||
# Relax atol when we have zeropoints since the way machete applies
|
||||
# zeropoints (after scales) causes noise around 0
|
||||
atol = 1 if zero_points else min(5e-2 * math.sqrt(k), 1)
|
||||
torch.testing.assert_close(output, output_ref, rtol=1e-1, atol=atol)
|
||||
@@ -1,622 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the marlin kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_marlin_gemm.py`.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.int8_utils import (
|
||||
per_token_quant_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
get_marlin_workspace,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_permute_bias,
|
||||
query_marlin_supported_quant_types,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
rand_marlin_weight_mxfp4_like,
|
||||
rand_marlin_weight_nvfp4_like,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
marlin_quant_fp8_torch,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
|
||||
awq_marlin_quantize,
|
||||
get_weight_perm,
|
||||
marlin_quantize,
|
||||
marlin_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
awq_pack,
|
||||
gptq_pack,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"These tests require marlin, which is not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
USE_ATOMIC_ADD_OPTS = [False, True]
|
||||
USE_FP32_REDUCE_OPTS = [True]
|
||||
|
||||
MARLIN_K_CHUNKS = [128]
|
||||
MARLIN_N_CHUNKS = [64, 256]
|
||||
|
||||
MARLIN_REPACK_NK_FACTORS = [
|
||||
(4, 8),
|
||||
(7, 5),
|
||||
(13, 11),
|
||||
]
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 1, 1),
|
||||
(1, 4, 8),
|
||||
(26, 37, 13),
|
||||
(257, 13, 11),
|
||||
]
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
|
||||
DENSE_MARLIN_QUANT_TEST_CONFIGS = [
|
||||
# AWQ-INT4
|
||||
{"b_type": scalar_types.uint4, "group_blocks": [-1, 2, 4, 8]},
|
||||
# GPTQ-INT4
|
||||
{
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT8
|
||||
{
|
||||
"b_type": scalar_types.uint8b128,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# FP8
|
||||
{"b_type": scalar_types.float8_e4m3fn, "group_blocks": [-1, 8]},
|
||||
# NVFP4
|
||||
{"b_type": scalar_types.float4_e2m1f, "group_blocks": [1]},
|
||||
# MXFP4
|
||||
{
|
||||
"a_type": [scalar_types.bfloat16],
|
||||
"b_type": scalar_types.float4_e2m1f,
|
||||
"group_blocks": [2],
|
||||
},
|
||||
# AWQ-INT4 with INT8 activation
|
||||
{
|
||||
"a_type": [scalar_types.int8],
|
||||
"b_type": scalar_types.uint4,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT4 with INT8 activation
|
||||
{
|
||||
"a_type": [scalar_types.int8],
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# GPTQ-INT4 with FP8 activation
|
||||
{
|
||||
"a_type": [scalar_types.float8_e4m3fn],
|
||||
"b_type": scalar_types.uint4b8,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# AWQ-INT4 with FP8 activation
|
||||
{
|
||||
"a_type": [scalar_types.float8_e4m3fn],
|
||||
"b_type": scalar_types.uint4,
|
||||
"group_blocks": [-1, 2, 4, 8],
|
||||
},
|
||||
# MXFP4 with FP8 activation
|
||||
{
|
||||
"a_type": [scalar_types.float8_e4m3fn],
|
||||
"b_type": scalar_types.float4_e2m1f,
|
||||
"c_type": [scalar_types.bfloat16],
|
||||
"group_blocks": [2],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def compute_max_diff(output, output_ref):
|
||||
return torch.mean(torch.abs(output - output_ref)) / torch.mean(
|
||||
torch.abs(output_ref)
|
||||
)
|
||||
|
||||
|
||||
def rand_data(shape, dtype=torch.float16):
|
||||
return torch.randn(shape, dtype=dtype, device="cuda")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
def test_marlin_int4_fp8_preprocess_without_zp():
|
||||
qweight_unpacked = torch.randint(
|
||||
0, 16, size=(2048, 2048), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
qweight_packed = qweight_unpacked[:, ::2] * 16 + qweight_unpacked[:, 1::2]
|
||||
qweight_packed = qweight_packed.to(torch.int8).view(torch.int32)
|
||||
|
||||
cuda_res = ops.marlin_int4_fp8_preprocess(qweight_packed)
|
||||
|
||||
torch_res = torch.where(
|
||||
qweight_unpacked >= 8, qweight_unpacked - 8, 15 - qweight_unpacked
|
||||
)
|
||||
torch_res = torch_res[:, ::2] * 16 + torch_res[:, 1::2]
|
||||
torch_res = torch_res.to(torch.int8).view(torch.int32)
|
||||
|
||||
assert (cuda_res == torch_res).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
def test_marlin_int4_fp8_preprocess_awq():
|
||||
group_size = 128
|
||||
|
||||
qweight_unpacked = torch.randint(
|
||||
0, 16, size=(2048, 2048), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
qzeros_unpacked = torch.randint(
|
||||
0, 16, size=(2048 // group_size, 2048), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
qweight_packed = qweight_unpacked[:, ::2] * 16 + qweight_unpacked[:, 1::2]
|
||||
qweight_packed = qweight_packed.to(torch.int8).view(torch.int32)
|
||||
qzeros_packed = qzeros_unpacked[:, ::2] * 16 + qzeros_unpacked[:, 1::2]
|
||||
qzeros_packed = qzeros_packed.to(torch.int8).view(torch.int32)
|
||||
|
||||
cuda_res = ops.marlin_int4_fp8_preprocess(qweight_packed, qzeros_packed)
|
||||
|
||||
repeated_zp = qzeros_unpacked.repeat_interleave(group_size, 0)
|
||||
torch_res = qweight_unpacked - repeated_zp
|
||||
torch_res[torch_res < 0] = 15 - qweight_unpacked[torch_res < 0]
|
||||
torch_res = torch_res[:, ::2] * 16 + torch_res[:, 1::2]
|
||||
torch_res = torch_res.to(torch.int8).view(torch.int32)
|
||||
|
||||
assert (cuda_res == torch_res).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
|
||||
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
|
||||
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(False, False))
|
||||
@pytest.mark.parametrize("is_a_8bit", [True, False])
|
||||
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
|
||||
def test_gptq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
|
||||
n_factor, k_factor = nk_factors
|
||||
|
||||
size_k = k_chunk * k_factor
|
||||
size_n = n_chunk * n_factor
|
||||
group_size = 128
|
||||
|
||||
# Normalize group_size
|
||||
if group_size == -1:
|
||||
group_size = size_k
|
||||
assert group_size <= size_k
|
||||
|
||||
# Create input
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
# Quantize
|
||||
_, q_w, _ = gptq_quantize_weights(b_weight, quant_type, group_size)
|
||||
|
||||
# Pack to GPTQ format
|
||||
q_w_gptq = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
# Pack to Marlin format
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w_1 = marlin_weights(
|
||||
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit
|
||||
)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.gptq_marlin_repack,
|
||||
(q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
)
|
||||
|
||||
# Run Marlin repack GPU kernel
|
||||
marlin_q_w_2 = ops.gptq_marlin_repack(
|
||||
q_w_gptq, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("k_chunk", MARLIN_K_CHUNKS)
|
||||
@pytest.mark.parametrize("n_chunk", MARLIN_N_CHUNKS)
|
||||
@pytest.mark.parametrize("quant_type", query_marlin_supported_quant_types(True))
|
||||
@pytest.mark.parametrize("is_a_8bit", [True, False])
|
||||
@pytest.mark.parametrize("nk_factors", MARLIN_REPACK_NK_FACTORS)
|
||||
def test_awq_marlin_repack(k_chunk, n_chunk, quant_type, is_a_8bit, nk_factors):
|
||||
n_factor, k_factor = nk_factors
|
||||
|
||||
size_k = k_chunk * k_factor
|
||||
size_n = n_chunk * n_factor
|
||||
|
||||
group_size = 128
|
||||
|
||||
# Create input
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
# Quantize
|
||||
w_ref, q_w, s, zp = quantize_weights(
|
||||
b_weight, quant_type, group_size, zero_points=True
|
||||
)
|
||||
|
||||
# Pack to AWQ format
|
||||
q_w_awq = awq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
# Pack to Marlin format
|
||||
weight_perm = get_weight_perm(quant_type.size_bits, is_a_8bit)
|
||||
marlin_q_w_1 = marlin_weights(
|
||||
q_w, size_k, size_n, quant_type.size_bits, weight_perm, is_a_8bit
|
||||
)
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.awq_marlin_repack,
|
||||
(q_w_awq, size_k, size_n, quant_type.size_bits, is_a_8bit),
|
||||
)
|
||||
|
||||
# Run Marlin repack GPU kernel
|
||||
marlin_q_w_2 = ops.awq_marlin_repack(
|
||||
q_w_awq, size_k, size_n, quant_type.size_bits, is_a_8bit
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(marlin_q_w_1, marlin_q_w_2)
|
||||
|
||||
|
||||
def marlin_generate_valid_test_cases():
|
||||
all_combinations = itertools.product(
|
||||
DENSE_MARLIN_QUANT_TEST_CONFIGS,
|
||||
MNK_FACTORS,
|
||||
MARLIN_N_CHUNKS,
|
||||
MARLIN_K_CHUNKS,
|
||||
USE_ATOMIC_ADD_OPTS,
|
||||
USE_FP32_REDUCE_OPTS,
|
||||
)
|
||||
|
||||
def is_invalid(
|
||||
a_type,
|
||||
b_type,
|
||||
c_type,
|
||||
group_blocks,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
if use_atomic_add:
|
||||
if use_fp32_reduce:
|
||||
return False
|
||||
if (
|
||||
c_type == scalar_types.bfloat16
|
||||
and torch.cuda.get_device_capability()[0] < 9
|
||||
):
|
||||
return False
|
||||
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
if group_size > 0 and size_k % group_size != 0:
|
||||
return False
|
||||
|
||||
if group_size == size_k:
|
||||
return False
|
||||
|
||||
return a_type.size_bits < 16 or a_type is c_type
|
||||
|
||||
cases = []
|
||||
for case in all_combinations:
|
||||
quant_test_config, mnk_factors, n_chunk, k_chunk, *_ = case
|
||||
size_m = mnk_factors[0]
|
||||
size_n = mnk_factors[1] * n_chunk
|
||||
size_k = mnk_factors[2] * k_chunk
|
||||
|
||||
f16_types = [scalar_types.float16, scalar_types.bfloat16]
|
||||
inner_combinations = itertools.product(
|
||||
quant_test_config.get("a_type", f16_types),
|
||||
[quant_test_config["b_type"]],
|
||||
quant_test_config.get("c_type", f16_types),
|
||||
quant_test_config["group_blocks"],
|
||||
) # type: ignore[call-overload]
|
||||
|
||||
for sub_case in inner_combinations:
|
||||
if (
|
||||
sub_case[0] == scalar_types.float8_e4m3fn
|
||||
and not current_platform.is_device_capability(89)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
):
|
||||
continue
|
||||
args = sub_case + (size_m, size_n, size_k) + case[4:]
|
||||
if is_invalid(*args):
|
||||
cases.append(args)
|
||||
return cases
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("gptq_marlin"),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"a_type, b_type, c_type, group_blocks,"
|
||||
"size_m, size_n, size_k,"
|
||||
"use_atomic_add, use_fp32_reduce"
|
||||
),
|
||||
marlin_generate_valid_test_cases(),
|
||||
)
|
||||
def test_marlin_gemm(
|
||||
a_type,
|
||||
b_type,
|
||||
c_type,
|
||||
group_blocks,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add,
|
||||
use_fp32_reduce,
|
||||
):
|
||||
has_zp = b_type in [scalar_types.uint4, scalar_types.uint8]
|
||||
|
||||
group_size = group_blocks if group_blocks <= 0 else group_blocks * 16
|
||||
|
||||
if c_type == scalar_types.float16:
|
||||
dtype = torch.float16
|
||||
elif c_type == scalar_types.bfloat16:
|
||||
dtype = torch.bfloat16
|
||||
else:
|
||||
raise RuntimeError("unsupported c_type")
|
||||
|
||||
if a_type == scalar_types.int8:
|
||||
a_dtype = torch.int8
|
||||
elif a_type == scalar_types.float8_e4m3fn:
|
||||
a_dtype = torch.float8_e4m3fn
|
||||
else:
|
||||
a_dtype = dtype
|
||||
|
||||
a_input = rand_data((size_m, size_k), dtype=dtype)
|
||||
b_weight = rand_data((size_k, size_n), dtype=dtype)
|
||||
|
||||
if b_type == scalar_types.float4_e2m1f:
|
||||
if group_size == 16:
|
||||
w_ref, marlin_q_w, marlin_s, marlin_s2 = rand_marlin_weight_nvfp4_like(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
else:
|
||||
w_ref, marlin_q_w, marlin_s = rand_marlin_weight_mxfp4_like(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
marlin_s2 = None
|
||||
|
||||
marlin_zp = None
|
||||
elif b_type == scalar_types.float8_e4m3fn:
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quant_fp8_torch(
|
||||
b_weight.T, group_size, input_dtype=a_dtype
|
||||
)
|
||||
marlin_zp = None
|
||||
marlin_s2 = None
|
||||
elif has_zp:
|
||||
w_ref, marlin_q_w, marlin_s, marlin_zp = awq_marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
marlin_s2 = None
|
||||
else:
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(
|
||||
b_weight, b_type, group_size, input_dtype=a_dtype
|
||||
)
|
||||
|
||||
marlin_zp = None
|
||||
marlin_s2 = None
|
||||
|
||||
workspace = marlin_make_workspace_new(w_ref.device)
|
||||
|
||||
if a_type == scalar_types.int8:
|
||||
a_input, a_scales = per_token_quant_int8(a_input)
|
||||
a_input_ref = a_input.to(a_scales.dtype) * a_scales.view(-1, 1)
|
||||
a_input_ref = a_input_ref.to(dtype)
|
||||
|
||||
if group_size != -1:
|
||||
a_scales = a_scales / 4096 * marlin_s.max()
|
||||
a_scales = a_scales.float()
|
||||
marlin_s = marlin_s / marlin_s.max() * 4096
|
||||
marlin_s = marlin_s.round().to(torch.int16).view(dtype)
|
||||
elif a_type == scalar_types.float8_e4m3fn:
|
||||
a_input, a_scales = ops.scaled_fp8_quant(a_input, use_per_token_if_dynamic=True)
|
||||
a_input_ref = a_input.to(a_scales.dtype) * a_scales.view(-1, 1)
|
||||
a_input_ref = a_input_ref.to(dtype)
|
||||
else:
|
||||
assert a_type.size_bits == 16
|
||||
a_input_ref = a_input
|
||||
a_scales = None
|
||||
|
||||
output = torch.empty((size_m, size_n), dtype=dtype, device=a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
a_input,
|
||||
output,
|
||||
marlin_q_w,
|
||||
None,
|
||||
marlin_s,
|
||||
a_scales,
|
||||
marlin_s2,
|
||||
marlin_zp,
|
||||
workspace,
|
||||
b_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
use_atomic_add=use_atomic_add,
|
||||
use_fp32_reduce=use_fp32_reduce,
|
||||
is_zp_float=False,
|
||||
)
|
||||
output_ref = torch.matmul(a_input_ref, w_ref)
|
||||
|
||||
max_diff = compute_max_diff(output, output_ref)
|
||||
assert max_diff < 0.04
|
||||
|
||||
|
||||
def test_marlin_persistent_locks_cuda_graph_streams(workspace_init):
|
||||
"""Compiled GEMMs select stream-local locks at runtime and survive replay."""
|
||||
size_m, size_k, size_n = 32, 1024, 2048
|
||||
inputs = [rand_data((size_m, size_k)) for _ in range(2)]
|
||||
weight = rand_data((size_k, size_n))
|
||||
quant_type = scalar_types.uint4b8
|
||||
w_ref, qweight, scales = marlin_quantize(weight, quant_type, 128)
|
||||
zeros = marlin_make_empty(scales.device)
|
||||
streams = [torch.cuda.Stream() for _ in inputs]
|
||||
graphs = [torch.cuda.CUDAGraph() for _ in inputs]
|
||||
locks = []
|
||||
outputs = []
|
||||
|
||||
@torch.compile(backend="eager", fullgraph=True)
|
||||
def run(x):
|
||||
return torch.ops.vllm.marlin_gemm(
|
||||
x,
|
||||
qweight,
|
||||
None,
|
||||
scales,
|
||||
None,
|
||||
None,
|
||||
zeros,
|
||||
None,
|
||||
quant_type.id,
|
||||
size_m,
|
||||
size_n,
|
||||
size_k,
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
)
|
||||
|
||||
for stream, graph, x in zip(streams, graphs, inputs):
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
locks.append(get_marlin_workspace(x.device))
|
||||
run(x)
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
outputs.append(run(x))
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
assert locks[0].data_ptr() != locks[1].data_ptr()
|
||||
current_workspace_manager().lock()
|
||||
for _ in range(3):
|
||||
for stream, graph, x, lock in zip(streams, graphs, inputs, locks):
|
||||
with torch.cuda.stream(stream):
|
||||
assert get_marlin_workspace(x.device) is lock
|
||||
x.add_(0.1)
|
||||
graph.replay()
|
||||
for stream in streams:
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
for x, output, lock in zip(inputs, outputs, locks):
|
||||
assert compute_max_diff(output, x @ w_ref) < 0.04
|
||||
assert torch.count_nonzero(lock) == 0
|
||||
|
||||
|
||||
def test_marlin_gemm_subset_input():
|
||||
quant_type = scalar_types.uint4b8
|
||||
group_size = 128
|
||||
|
||||
size_m, size_k, size_n = 32, 1024, 2048
|
||||
big_m = size_m * 2
|
||||
big_k = size_k * 2
|
||||
|
||||
a_input = rand_data((big_m, big_k))[8 : size_m + 8, 8 : size_k + 8]
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
a_input,
|
||||
None,
|
||||
marlin_q_w,
|
||||
None,
|
||||
marlin_s,
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
)
|
||||
output_ref = torch.matmul(a_input, w_ref)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
max_diff = compute_max_diff(output, output_ref)
|
||||
|
||||
assert max_diff < 0.04
|
||||
|
||||
|
||||
@pytest.mark.parametrize("size_m", [1, 256])
|
||||
def test_marlin_gemm_with_bias(size_m):
|
||||
quant_type = scalar_types.uint4b8
|
||||
group_size = 128
|
||||
|
||||
size_k, size_n = 1024, 2048
|
||||
a_input = rand_data((size_m, size_k))
|
||||
b_weight = rand_data((size_k, size_n))
|
||||
b_bias = rand_data((size_n,)) * 10
|
||||
|
||||
marlin_bias = marlin_permute_bias(b_bias)
|
||||
|
||||
w_ref, marlin_q_w, marlin_s = marlin_quantize(b_weight, quant_type, group_size)
|
||||
|
||||
marlin_zp = marlin_make_empty(marlin_s.device)
|
||||
workspace = marlin_make_workspace_new(a_input.device)
|
||||
|
||||
output = ops.marlin_gemm(
|
||||
a_input,
|
||||
None,
|
||||
marlin_q_w,
|
||||
marlin_bias,
|
||||
marlin_s,
|
||||
None,
|
||||
None,
|
||||
marlin_zp,
|
||||
workspace,
|
||||
quant_type,
|
||||
a_input.shape[0],
|
||||
b_weight.shape[1],
|
||||
a_input.shape[1],
|
||||
use_atomic_add=False,
|
||||
use_fp32_reduce=True,
|
||||
is_zp_float=False,
|
||||
)
|
||||
output_ref = torch.matmul(a_input, w_ref) + b_bias.view(1, -1)
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
max_diff = compute_max_diff(output, output_ref)
|
||||
|
||||
assert max_diff < 0.04
|
||||
@@ -1,851 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Marlin thread-tile padding of TP-sharded weight shapes.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_marlin_tile_padding.py`.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
get_marlin_workspace,
|
||||
marlin_make_empty,
|
||||
marlin_make_workspace_new,
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_pad_qweight,
|
||||
marlin_pad_scales,
|
||||
marlin_padded_nk,
|
||||
marlin_permute_scales,
|
||||
marlin_repacked_nk,
|
||||
marlin_zero_points,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
apply_fp4_marlin_linear,
|
||||
is_fp4_marlin_supported,
|
||||
prepare_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
apply_fp8_marlin_linear,
|
||||
apply_mxfp8_marlin_linear,
|
||||
is_fp8_marlin_supported,
|
||||
prepare_fp8_layer_for_marlin,
|
||||
prepare_mxfp8_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
gptq_pack,
|
||||
gptq_quantize_weights,
|
||||
quantize_weights,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
# (size_n, size_k) rank-local shapes that violate Marlin tile alignment,
|
||||
# e.g. produced by TP-sharding dims that are valid at TP=1.
|
||||
ODD_SHAPES = [
|
||||
(200, 288), # N padded
|
||||
(256, 208), # K padded
|
||||
(200, 208), # both padded
|
||||
(4640, 512), # Nemotron-Super-120B q_proj shard at TP=4
|
||||
]
|
||||
ALIGNED_SHAPES = [(64, 128), (128, 64), (256, 256), (4608, 4096)]
|
||||
|
||||
|
||||
def _is_tile_aligned(size_n: int, size_k: int) -> bool:
|
||||
return (size_n % 64 == 0 and size_k % 128 == 0) or (
|
||||
size_n % 128 == 0 and size_k % 64 == 0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", ODD_SHAPES + ALIGNED_SHAPES)
|
||||
@pytest.mark.parametrize("group_size", [-1, 16, 32, 64, 128])
|
||||
def test_marlin_padded_nk(shape, group_size):
|
||||
size_n, size_k = shape
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
|
||||
assert padded_n >= size_n and padded_k >= size_k
|
||||
assert _is_tile_aligned(padded_n, padded_k)
|
||||
if group_size > 0:
|
||||
assert padded_k % group_size == 0
|
||||
|
||||
# Aligned shapes must pass through unchanged (zero hot-path cost).
|
||||
if _is_tile_aligned(size_n, size_k) and (
|
||||
group_size <= 0 or size_k % group_size == 0
|
||||
):
|
||||
assert (padded_n, padded_k) == (size_n, size_k)
|
||||
|
||||
# Minimal: no valid shape with a smaller padded area exists.
|
||||
area = padded_n * padded_k
|
||||
for cand_n in range(size_n, padded_n + 1):
|
||||
for cand_k in range(size_k, padded_k + 1):
|
||||
if (
|
||||
_is_tile_aligned(cand_n, cand_k)
|
||||
and (group_size <= 0 or cand_k % group_size == 0)
|
||||
and cand_n * cand_k < area
|
||||
):
|
||||
pytest.fail(f"({cand_n}, {cand_k}) beats ({padded_n}, {padded_k})")
|
||||
|
||||
# Apply-time derivation from the repacked-tensor shape must round-trip.
|
||||
for num_bits in (4, 8):
|
||||
pack_factor = 32 // num_bits
|
||||
repacked_shape = (
|
||||
padded_k // GPTQ_MARLIN_TILE,
|
||||
padded_n * GPTQ_MARLIN_TILE // pack_factor,
|
||||
)
|
||||
repacked = torch.empty(repacked_shape, device="meta")
|
||||
assert marlin_repacked_nk(repacked, num_bits) == (padded_n, padded_k)
|
||||
|
||||
|
||||
def test_marlin_pad_helpers_shapes():
|
||||
size_n, size_k, group_size = 200, 208, 16
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
|
||||
qweight = torch.zeros(size_k // 8, size_n, dtype=torch.int32)
|
||||
padded = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
assert padded.shape == (padded_k // 8, padded_n)
|
||||
|
||||
scales = torch.ones(size_k // group_size, size_n)
|
||||
padded = marlin_pad_scales(scales, size_n, size_k, padded_n, padded_k, group_size)
|
||||
assert padded.shape == (padded_k // group_size, padded_n)
|
||||
assert padded[:, size_n:].abs().sum() == 0
|
||||
|
||||
channelwise = torch.ones(1, size_n)
|
||||
padded = marlin_pad_scales(channelwise, size_n, size_k, padded_n, padded_k, -1)
|
||||
assert padded.shape == (1, padded_n)
|
||||
|
||||
|
||||
# Rank-local MoE intermediate sizes. group<=0 / 32 with a non-multiple-of-64
|
||||
# size is where tile padding triggers; 64/128 are already tile-aligned.
|
||||
MOE_INTERMEDIATE_SIZES = [64, 96, 100, 176, 192, 256, 2816]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", MOE_INTERMEDIATE_SIZES)
|
||||
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128])
|
||||
def test_marlin_moe_padded_intermediate(intermediate, group_size):
|
||||
# The MoE gate only admits shapes where the group does not straddle the
|
||||
# boundary, i.e. group divides the intermediate size.
|
||||
if group_size > 0 and intermediate % group_size != 0:
|
||||
pytest.skip("group straddles the boundary; rejected by the MoE gate")
|
||||
|
||||
padded = marlin_moe_padded_intermediate(intermediate, group_size)
|
||||
assert padded >= intermediate
|
||||
# Valid MoE thread tile: gate-up n = 2*intermediate % 128, down k % 64.
|
||||
assert (2 * padded) % 128 == 0
|
||||
assert padded % 64 == 0
|
||||
if group_size > 0:
|
||||
assert padded % group_size == 0
|
||||
|
||||
# Minimal: no smaller valid intermediate exists.
|
||||
for cand in range(intermediate, padded):
|
||||
if (
|
||||
(2 * cand) % 128 == 0
|
||||
and cand % 64 == 0
|
||||
and (group_size <= 0 or cand % group_size == 0)
|
||||
):
|
||||
pytest.fail(f"{cand} beats {padded}")
|
||||
|
||||
# Already-tile-aligned sizes pass through unchanged (zero hot-path cost).
|
||||
if intermediate % 64 == 0:
|
||||
assert padded == intermediate
|
||||
|
||||
|
||||
def test_marlin_moe_pad_helpers_shapes():
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
_pad_rows,
|
||||
_pad_w13_bias,
|
||||
_pad_w13_shard_cols,
|
||||
)
|
||||
|
||||
E, rows, N, padded_N = 2, 8, 96, 128
|
||||
|
||||
# w13 stores the two gate/up shards along the last dim; padding each shard
|
||||
# must preserve the loaded values and zero the padded columns.
|
||||
w13 = torch.arange(E * rows * 2 * N).reshape(E, rows, 2 * N).float()
|
||||
padded = _pad_w13_shard_cols(w13, N, padded_N)
|
||||
assert padded.shape == (E, rows, 2 * padded_N)
|
||||
shards = padded.view(E, rows, 2, padded_N)
|
||||
orig = w13.view(E, rows, 2, N)
|
||||
assert torch.equal(shards[..., :N], orig)
|
||||
assert shards[..., N:].abs().sum() == 0
|
||||
|
||||
# w2 stores the intermediate dim in the rows.
|
||||
w2 = torch.ones(E, N // 32, 16)
|
||||
padded = _pad_rows(w2, padded_N // 32)
|
||||
assert padded.shape == (E, padded_N // 32, 16)
|
||||
assert padded[:, N // 32 :, :].abs().sum() == 0
|
||||
|
||||
bias = torch.arange(E * 2 * N).reshape(E, 2 * N).float()
|
||||
padded = _pad_w13_bias(bias, N, padded_N)
|
||||
assert padded.shape == (E, 2 * padded_N)
|
||||
bias_shards = padded.view(E, 2, padded_N)
|
||||
assert torch.equal(bias_shards[..., :N], bias.view(E, 2, N))
|
||||
assert bias_shards[..., N:].abs().sum() == 0
|
||||
|
||||
|
||||
def _gpu_marlin_unsupported() -> bool:
|
||||
return not (
|
||||
current_platform.is_cuda() and current_platform.has_device_capability(80)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", ODD_SHAPES)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_fp8_marlin_padded_round_trip(shape, use_bias):
|
||||
size_n, size_k = shape
|
||||
dtype = torch.float16
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = size_n
|
||||
layer.input_size_per_partition = size_k
|
||||
layer.orig_dtype = dtype
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device="cuda") / size_k**0.5
|
||||
scale = weight.abs().max() / 448
|
||||
weight_fp8 = (weight / scale).to(torch.float8_e4m3fn)
|
||||
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
scale.to(torch.float32), requires_grad=False
|
||||
)
|
||||
bias = None
|
||||
if use_bias:
|
||||
bias = torch.randn(size_n, dtype=dtype, device="cuda")
|
||||
layer.bias = torch.nn.Parameter(bias.clone(), requires_grad=False)
|
||||
|
||||
prepare_fp8_layer_for_marlin(layer, size_k_first=True)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device="cuda")
|
||||
output = apply_fp8_marlin_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
bias=layer.bias if use_bias else None,
|
||||
)
|
||||
ref = x @ (weight_fp8.to(dtype) * scale.to(dtype))
|
||||
if use_bias:
|
||||
ref = ref + bias
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def _dequant_fp4(packed: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
|
||||
"""Dequantize packed e2m1 nibbles (N, K // 2) -> (N, K) in dtype."""
|
||||
lo = (packed & 0b10000000) | ((packed & 0b01110000) >> 2)
|
||||
lo = lo.view(torch.float8_e4m3fn).to(dtype) * (2**6)
|
||||
hi_bits = packed << 4
|
||||
hi = (hi_bits & 0b10000000) | ((hi_bits & 0b01110000) >> 2)
|
||||
hi = hi.view(torch.float8_e4m3fn).to(dtype) * (2**6)
|
||||
return torch.cat([hi.unsqueeze(2), lo.unsqueeze(2)], 2).view(packed.size(0), -1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp4_marlin_supported(),
|
||||
reason="FP4 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", ODD_SHAPES)
|
||||
def test_nvfp4_marlin_padded_round_trip(shape):
|
||||
size_n, size_k = shape
|
||||
group_size = 16
|
||||
dtype = torch.float16
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = size_n
|
||||
layer.input_size_per_partition = size_k
|
||||
layer.params_dtype = dtype
|
||||
|
||||
packed = torch.randint(
|
||||
0, 256, (size_n, size_k // 2), dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
scales = (torch.rand(size_n, size_k // group_size, device="cuda") + 0.25).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
global_scale = torch.tensor([0.002], dtype=torch.float32, device="cuda")
|
||||
|
||||
ref_weight = (
|
||||
_dequant_fp4(packed, dtype)
|
||||
* scales.to(dtype).repeat_interleave(group_size, 1)
|
||||
* global_scale.to(dtype)
|
||||
)
|
||||
|
||||
layer.weight = torch.nn.Parameter(packed, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
|
||||
layer.weight_global_scale = torch.nn.Parameter(global_scale, requires_grad=False)
|
||||
|
||||
prepare_fp4_layer_for_marlin(layer)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device="cuda") / size_k**0.5
|
||||
output = apply_fp4_marlin_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
weight_global_scale=layer.weight_global_scale,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
)
|
||||
ref = x @ ref_weight.T
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported(),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", ODD_SHAPES)
|
||||
@pytest.mark.parametrize("group_size", [-1, 128])
|
||||
def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
"""Pad-then-repack a GPTQ int4 weight the way MarlinLinearKernel does and
|
||||
check the GEMM against the dequantized reference.
|
||||
|
||||
Symmetric int4's quantized zero decodes to -8, so this exercises the
|
||||
zero-padded-scales cancellation, not just zero weights.
|
||||
"""
|
||||
size_n, size_k = shape
|
||||
if group_size > 0 and size_k % group_size != 0:
|
||||
pytest.skip("group must divide the rank-local K (not fixable by padding)")
|
||||
dtype = torch.float16
|
||||
quant_type = scalar_types.uint4b8
|
||||
device = torch.device("cuda")
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
|
||||
w_ref, q_w, s = gptq_quantize_weights(weight, quant_type, group_size)
|
||||
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
)
|
||||
s = marlin_pad_scales(s, size_n, size_k, padded_n, padded_k, group_size)
|
||||
marlin_s = marlin_permute_scales(
|
||||
s, size_k=padded_k, size_n=padded_n, group_size=group_size
|
||||
)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device=device)
|
||||
output = apply_gptq_marlin_linear(
|
||||
input=x,
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_make_empty(device),
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
|
||||
def test_fp8_block_marlin_padded_round_trip(shape):
|
||||
"""Block-quantized FP8 (e.g. Nemotron NVFP4 checkpoints' FP8 layers):
|
||||
group_size=128 exercises the lcm K-alignment in marlin_padded_nk and the
|
||||
weight_scale_inv group-wise scale padding."""
|
||||
size_n, size_k = shape
|
||||
block = 128
|
||||
dtype = torch.float16
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = size_n
|
||||
layer.input_size_per_partition = size_k
|
||||
layer.orig_dtype = dtype
|
||||
layer.weight_block_size = [block, block]
|
||||
|
||||
weight = torch.randn(size_n, size_k, dtype=dtype, device="cuda") / size_k**0.5
|
||||
n_blocks, k_blocks = (size_n + block - 1) // block, size_k // block
|
||||
padded = torch.zeros(n_blocks * block, size_k, dtype=dtype, device="cuda")
|
||||
padded[:size_n] = weight
|
||||
scales = padded.view(n_blocks, block, k_blocks, block).abs().amax(dim=(1, 3)) / 448
|
||||
scales_expanded = scales.repeat_interleave(block, 0)[:size_n].repeat_interleave(
|
||||
block, 1
|
||||
)
|
||||
weight_fp8 = (weight / scales_expanded).to(torch.float8_e4m3fn)
|
||||
|
||||
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
|
||||
layer.weight_scale_inv = torch.nn.Parameter(
|
||||
scales.to(torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
prepare_fp8_layer_for_marlin(layer, size_k_first=False)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device="cuda")
|
||||
output = apply_fp8_marlin_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale_inv,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
bias=None,
|
||||
)
|
||||
ref = x @ (weight_fp8.to(dtype) * scales_expanded.to(dtype)).T
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"shape", [(32, 256), (96, 256), (130, 256), (200, 288), (4640, 512)]
|
||||
)
|
||||
def test_mxfp8_marlin_padded_round_trip(shape):
|
||||
"""MXFP8 exercises the e8m0 scale path, where padded 0.0 scales clamp to
|
||||
2^-127 instead of zero and must still contribute nothing."""
|
||||
size_n, size_k = shape
|
||||
group_size = 32
|
||||
# The e8m0-scale Marlin kernels are only instantiated for bf16 activations.
|
||||
dtype = torch.bfloat16
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = size_n
|
||||
layer.input_size_per_partition = size_k
|
||||
|
||||
weight_fp8 = (torch.randn(size_n, size_k, dtype=dtype, device="cuda") / 4).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
# e8m0 exponents around 1.0 (127): scales in [2^-6, 2^0]
|
||||
scales = torch.randint(
|
||||
121, 128, (size_n, size_k // group_size), dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
ref_weight = weight_fp8.to(dtype) * (
|
||||
2.0 ** (scales.to(dtype) - 127)
|
||||
).repeat_interleave(group_size, 1)
|
||||
|
||||
layer.weight = torch.nn.Parameter(weight_fp8, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(scales, requires_grad=False)
|
||||
|
||||
prepare_mxfp8_layer_for_marlin(layer)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device="cuda") / size_k**0.5
|
||||
output = apply_mxfp8_marlin_linear(
|
||||
input=x,
|
||||
weight=layer.weight,
|
||||
weight_scale=layer.weight_scale,
|
||||
workspace=get_marlin_workspace(layer.weight.device),
|
||||
size_n=size_n,
|
||||
size_k=size_k,
|
||||
)
|
||||
ref = x @ ref_weight.T
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported(),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
|
||||
def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
"""AWQ-style uint4 with runtime zero-points, padded the way
|
||||
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0."""
|
||||
size_n, size_k = shape
|
||||
group_size = 128
|
||||
dtype = torch.float16
|
||||
quant_type = scalar_types.uint4
|
||||
device = torch.device("cuda")
|
||||
|
||||
weight = torch.randn(size_k, size_n, dtype=dtype, device=device) / size_k**0.5
|
||||
w_ref, q_w, s, zp = quantize_weights(
|
||||
weight, quant_type, group_size, zero_points=True
|
||||
)
|
||||
qweight = gptq_pack(q_w, quant_type.size_bits, size_k, size_n)
|
||||
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, group_size)
|
||||
qweight = marlin_pad_qweight(qweight, size_n, size_k, padded_n, padded_k)
|
||||
marlin_qweight = ops.gptq_marlin_repack(
|
||||
b_q_weight=qweight,
|
||||
size_k=padded_k,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
)
|
||||
s = marlin_pad_scales(s, size_n, size_k, padded_n, padded_k, group_size)
|
||||
marlin_s = marlin_permute_scales(
|
||||
s, size_k=padded_k, size_n=padded_n, group_size=group_size
|
||||
)
|
||||
zp = marlin_pad_scales(zp, size_n, size_k, padded_n, padded_k, group_size)
|
||||
marlin_zp = marlin_zero_points(
|
||||
zp,
|
||||
size_k=padded_k // group_size,
|
||||
size_n=padded_n,
|
||||
num_bits=quant_type.size_bits,
|
||||
)
|
||||
|
||||
x = torch.randn(8, size_k, dtype=dtype, device=device)
|
||||
output = apply_gptq_marlin_linear(
|
||||
input=x,
|
||||
weight=marlin_qweight,
|
||||
weight_scale=marlin_s,
|
||||
weight_zp=marlin_zp,
|
||||
workspace=marlin_make_workspace_new(device),
|
||||
wtype=quant_type,
|
||||
output_size_per_partition=size_n,
|
||||
input_size_per_partition=size_k,
|
||||
)
|
||||
ref = x @ w_ref
|
||||
|
||||
assert output.shape == (8, size_n)
|
||||
torch.testing.assert_close(output, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
class _FakeLinear:
|
||||
def __init__(self, size_n, size_k, input_size=None):
|
||||
self.output_size_per_partition = size_n
|
||||
self.input_size_per_partition = size_k
|
||||
self.output_size = size_n
|
||||
self.input_size = input_size if input_size is not None else size_k
|
||||
|
||||
|
||||
def test_check_marlin_supports_layer_allow_tile_padding():
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_marlin_supports_layer,
|
||||
)
|
||||
|
||||
# Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding
|
||||
layer: LinearBase = _FakeLinear(4640, 512, input_size=2048)
|
||||
assert not check_marlin_supports_layer(layer, 128)
|
||||
assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True)
|
||||
|
||||
# A group straddling the TP shard cannot be fixed by padding
|
||||
layer = _FakeLinear(4608, 4672, input_size=18688)
|
||||
assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported(),
|
||||
reason="Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("group_size", [-1, 32])
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_gptq_marlin_moe_padded_round_trip(shape, group_size):
|
||||
"""Pad a tile-misaligned MoE intermediate the way the WNA16 Marlin MoE prep
|
||||
does, run the real repack + fused_marlin_moe, and check against the
|
||||
dequantized reference. Symmetric int4's quantized zero decodes to -8, so the
|
||||
padded region only stays out of the output via the zero-padded scales.
|
||||
"""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
_pad_rows,
|
||||
_pad_w13_shard_cols,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_padded_intermediate,
|
||||
marlin_moe_permute_scales,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m = 2, 33
|
||||
padded_n = marlin_moe_padded_intermediate(n, group_size)
|
||||
assert padded_n != n, "test should exercise padding"
|
||||
|
||||
dtype = torch.float16
|
||||
device = torch.device("cuda")
|
||||
quant_type = scalar_types.uint4b8
|
||||
bits = quant_type.size_bits
|
||||
pack = 32 // bits
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
|
||||
def quant(w, size_k, size_n):
|
||||
# w is (size_n, size_k); gptq expects (size_k, size_n).
|
||||
ref, q_w, s = gptq_quantize_weights(w.T, quant_type, group_size)
|
||||
return ref, gptq_pack(q_w, bits, size_k, size_n), s
|
||||
|
||||
w13_qw, w13_s, w13_ref = [], [], []
|
||||
w2_qw, w2_s, w2_ref = [], [], []
|
||||
for i in range(e):
|
||||
ref, qw, s = quant(w1[i], k, 2 * n)
|
||||
w13_ref.append(ref.T) # (2n, k)
|
||||
w13_qw.append(qw)
|
||||
w13_s.append(s)
|
||||
ref, qw, s = quant(w2[i], n, k)
|
||||
w2_ref.append(ref.T) # (k, n)
|
||||
w2_qw.append(qw)
|
||||
w2_s.append(s)
|
||||
|
||||
w13_qweight = torch.stack(w13_qw)
|
||||
w2_qweight = torch.stack(w2_qw)
|
||||
w13_scales = torch.stack(w13_s)
|
||||
w2_scales = torch.stack(w2_s)
|
||||
w1_ref = torch.stack(w13_ref) # (e, 2n, k)
|
||||
w2_ref = torch.stack(w2_ref) # (e, k, n)
|
||||
|
||||
# Pad the intermediate via the production helpers.
|
||||
w13_qweight = _pad_w13_shard_cols(w13_qweight, n, padded_n)
|
||||
w2_qweight = _pad_rows(w2_qweight, padded_n // pack)
|
||||
w13_scales = _pad_w13_shard_cols(w13_scales, n, padded_n)
|
||||
if group_size > 0:
|
||||
w2_scales = _pad_rows(w2_scales, padded_n // group_size)
|
||||
|
||||
marlin_w13 = ops.gptq_marlin_moe_repack(
|
||||
w13_qweight, w13_qweight.shape[1] * pack, w13_qweight.shape[2], bits
|
||||
)
|
||||
marlin_w2 = ops.gptq_marlin_moe_repack(
|
||||
w2_qweight, w2_qweight.shape[1] * pack, w2_qweight.shape[2], bits
|
||||
)
|
||||
group_or_pack = group_size if group_size != -1 else pack
|
||||
marlin_w13_s = marlin_moe_permute_scales(
|
||||
s=w13_scales, size_k=n, size_n=w13_scales.shape[2], group_size=group_size
|
||||
)
|
||||
marlin_w2_s = marlin_moe_permute_scales(
|
||||
s=w2_scales,
|
||||
size_k=w2_scales.shape[1] * group_or_pack,
|
||||
size_n=w2_scales.shape[2],
|
||||
group_size=group_size,
|
||||
)
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
|
||||
marlin_out = fused_marlin_moe(
|
||||
a,
|
||||
marlin_w13,
|
||||
marlin_w2,
|
||||
None,
|
||||
None,
|
||||
marlin_w13_s,
|
||||
marlin_w2_s,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=quant_type.id,
|
||||
global_num_experts=e,
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
w1_ref,
|
||||
w2_ref,
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(marlin_out, ref, atol=5e-2, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="MoE Marlin is not selected on ROCm.",
|
||||
)
|
||||
def test_check_moe_marlin_supports_layer_padding():
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
check_moe_marlin_supports_layer,
|
||||
)
|
||||
|
||||
def make_layer(hidden, intermediate):
|
||||
layer = SimpleNamespace()
|
||||
layer.hidden_size = hidden
|
||||
layer.apply_router_weight_on_input = False
|
||||
layer.moe_config = SimpleNamespace(
|
||||
hidden_dim=hidden, intermediate_size_per_partition_unpadded=intermediate
|
||||
)
|
||||
return layer
|
||||
|
||||
# group=32 with intermediate % 64 != 0: rejected strictly, accepted w/ padding
|
||||
layer = make_layer(4096, 96)
|
||||
assert not check_moe_marlin_supports_layer(layer, 32)
|
||||
assert check_moe_marlin_supports_layer(layer, 32, allow_tile_padding=True)
|
||||
# channelwise misaligned intermediate is paddable
|
||||
assert check_moe_marlin_supports_layer(layer, -1, allow_tile_padding=True)
|
||||
|
||||
# A group straddling the boundary cannot be fixed by padding
|
||||
layer = make_layer(4096, 176)
|
||||
assert not check_moe_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
|
||||
# hidden_size is the MoE I/O extent and is never padded
|
||||
layer = make_layer(4090, 128)
|
||||
assert not check_moe_marlin_supports_layer(layer, 64, allow_tile_padding=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("quant", ["channel", "tensor"])
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
|
||||
real prepare + fused_marlin_moe against the dequantized reference."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_moe_padded_intermediate,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m = 2, 33
|
||||
fp8 = torch.float8_e4m3fn
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
padded_n = marlin_moe_padded_intermediate(n, -1)
|
||||
assert padded_n != n
|
||||
|
||||
def q(w): # (out, in) -> fp8 weight, scale, dequant reference
|
||||
dim = None if quant == "tensor" else 1
|
||||
s = (w.abs().amax(dim, keepdim=dim is not None) / 448.0).clamp(min=1e-8)
|
||||
wq = (w / s).clamp(-448, 448).to(fp8)
|
||||
ref = wq.to(dtype) * s.to(dtype)
|
||||
s = s.reshape(1) if quant == "tensor" else s.squeeze(1)
|
||||
return wq, s, ref
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w1 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
w13_q, w13_s, w1_ref = zip(*(q(w1[i]) for i in range(e)))
|
||||
w2_q, w2_s, w2_ref = zip(*(q(w2[i]) for i in range(e)))
|
||||
|
||||
w13_weight, w2_weight = torch.stack(w13_q), torch.stack(w2_q)
|
||||
layer = SimpleNamespace(
|
||||
num_experts=e,
|
||||
hidden_size=k,
|
||||
intermediate_size_per_partition=n,
|
||||
orig_dtype=dtype,
|
||||
w13_weight=w13_weight,
|
||||
)
|
||||
pw13, pw2, ps13, ps2 = prepare_fp8_moe_layer_for_marlin(
|
||||
layer, w13_weight, w2_weight, torch.stack(w13_s), torch.stack(w2_s)
|
||||
)
|
||||
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
out = fused_marlin_moe(
|
||||
a,
|
||||
pw13,
|
||||
pw2,
|
||||
None,
|
||||
None,
|
||||
ps13,
|
||||
ps2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
workspace=get_marlin_workspace(device),
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
torch.stack(w1_ref),
|
||||
torch.stack(w2_ref),
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_gpu_marlin_unsupported() or not is_fp8_marlin_supported(),
|
||||
reason="FP8 Marlin is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
|
||||
unit e8m0 scales so the reference is the exact fp8 dequant."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
fused_marlin_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
marlin_moe_intermediate_size,
|
||||
marlin_moe_padded_intermediate,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_mxfp8_moe_layer_for_marlin,
|
||||
)
|
||||
|
||||
n, k, e = shape
|
||||
topk, m, gs, e8m0_one = 2, 33, 32, 127
|
||||
fp8 = torch.float8_e4m3fn
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
padded_n = marlin_moe_padded_intermediate(n, gs)
|
||||
assert padded_n != n
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w13_weight = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / k**0.5
|
||||
w2_weight = torch.randn((e, k, n), device=device, dtype=dtype) / n**0.5
|
||||
w13_weight = w13_weight.clamp(-448, 448).to(fp8)
|
||||
w2_weight = w2_weight.clamp(-448, 448).to(fp8)
|
||||
w13_scale = torch.full(
|
||||
(e, 2 * n, k // gs), e8m0_one, dtype=torch.uint8, device=device
|
||||
)
|
||||
w2_scale = torch.full((e, k, n // gs), e8m0_one, dtype=torch.uint8, device=device)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
num_experts=e, hidden_size=k, intermediate_size_per_partition=n
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
pw13, pw2, ps13, ps2 = prepare_mxfp8_moe_layer_for_marlin(
|
||||
layer, w13_weight, w2_weight, w13_scale, w2_scale
|
||||
)
|
||||
assert marlin_moe_intermediate_size(pw13, pw2) == padded_n
|
||||
|
||||
score = torch.randn((m, e), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, False)
|
||||
out = fused_marlin_moe(
|
||||
a,
|
||||
pw13,
|
||||
pw2,
|
||||
None,
|
||||
None,
|
||||
ps13,
|
||||
ps2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
quant_type_id=scalar_types.float8_e4m3fn.id,
|
||||
global_num_experts=e,
|
||||
workspace=get_marlin_workspace(device),
|
||||
)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
ref = torch_experts(
|
||||
a,
|
||||
w13_weight.to(dtype),
|
||||
w2_weight.to(dtype),
|
||||
topk_weight=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=e,
|
||||
)
|
||||
torch.testing.assert_close(out, ref, atol=8e-2, rtol=0)
|
||||
@@ -1,322 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MXFP4 linear kernel selection logic (CPU-only).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
AiterMxfp4LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
FlashInferMxFp4LinearKernel,
|
||||
MarlinMxFp4LinearKernel,
|
||||
MxFp4LinearKernel,
|
||||
MxFp4LinearLayerConfig,
|
||||
XPUMxFp4LinearKernel,
|
||||
init_mxfp4_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E3M2Dynamic,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# Kernels that quantize activations themselves (true W4A4): they require an
|
||||
# explicit MXFP4-dynamic activation key.
|
||||
_TRUE_W4A4_KERNELS = [
|
||||
FlashInferMxFp4LinearKernel,
|
||||
XPUMxFp4LinearKernel,
|
||||
AiterMxfp4LinearKernel,
|
||||
]
|
||||
|
||||
# Weight-only (A16) kernels: they never quantize activations. They still accept
|
||||
# MXFP4 activation keys as an intentional compatibility fallback.
|
||||
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel]
|
||||
|
||||
|
||||
def _make_emulation_layer():
|
||||
layer = torch.nn.Module()
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(torch.ones((2, 2), dtype=torch.uint8), False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
torch.nn.Parameter(torch.ones((2, 1), dtype=torch.uint8), False),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def test_can_implement_is_abstract():
|
||||
"""Test that can_implement()/is_supported() are properly defined."""
|
||||
assert hasattr(MxFp4LinearKernel, "can_implement")
|
||||
assert hasattr(MxFp4LinearKernel, "is_supported")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
|
||||
def test_true_w4a4_kernels_accept_dynamic_mxfp4_activation(kernel_cls):
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert can_implement, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
|
||||
def test_true_w4a4_kernels_reject_unset_activation(kernel_cls):
|
||||
"""None means weight-only/unquantized activations, not dynamic MXFP4."""
|
||||
config = MxFp4LinearLayerConfig()
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
|
||||
def test_true_w4a4_kernels_reject_explicit_non_mxfp4_activation(kernel_cls):
|
||||
"""FlashInfer/XPU/Aiter quantize activations to MXFP4 internally, so an
|
||||
explicit request for a different activation format must be rejected."""
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _WEIGHT_ONLY_KERNELS)
|
||||
@pytest.mark.parametrize("activation_quant_key", [None, kMxfp4Dynamic])
|
||||
def test_weight_only_kernels_accept_unquantized_or_mxfp4_activation(
|
||||
kernel_cls, activation_quant_key
|
||||
):
|
||||
"""Marlin/Humming never quantize activations, so an unset activation key,
|
||||
or one that already describes MXFP4-shaped data, is tolerated. When an
|
||||
activation key is explicitly set, a warning must be logged noting that it
|
||||
is ignored, since these kernels are weight-only (A16)."""
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
|
||||
with patch(f"{kernel_cls.__module__}.logger.warning_once") as warning_once:
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert can_implement, reason
|
||||
|
||||
if activation_quant_key is None:
|
||||
warning_once.assert_not_called()
|
||||
else:
|
||||
warning_once.assert_called_once()
|
||||
message = warning_once.call_args.args[0]
|
||||
assert "the requested activation quantization" in message
|
||||
assert "is ignored" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _WEIGHT_ONLY_KERNELS)
|
||||
def test_weight_only_kernels_reject_non_mxfp4_activation(kernel_cls):
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation_quant_key",
|
||||
[None, kMxfp4Dynamic, kMxfp6E3M2Dynamic, kMxfp6E2M3Dynamic],
|
||||
)
|
||||
def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
|
||||
"""EmulationMxfp4LinearKernel is the universal fallback: it must accept
|
||||
every supported activation format."""
|
||||
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
|
||||
# which we are not testing here.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
|
||||
with patch(
|
||||
"vllm.model_executor.kernels.linear._get_linear_backend",
|
||||
return_value="emulation",
|
||||
):
|
||||
can_implement, reason = EmulationMxfp4LinearKernel.can_implement(config)
|
||||
assert can_implement, reason
|
||||
|
||||
|
||||
def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
|
||||
"""quant_dequant_func must be derived purely from the config's activation
|
||||
QuantKey, not set externally."""
|
||||
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
|
||||
# which we are not testing here.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.kernels.linear._get_linear_backend",
|
||||
return_value="emulation",
|
||||
):
|
||||
weight_only_config = MxFp4LinearLayerConfig()
|
||||
kernel = EmulationMxfp4LinearKernel(weight_only_config)
|
||||
x = torch.randn(4)
|
||||
# identity for weight-only
|
||||
assert torch.equal(kernel.quant_dequant_func(x), x)
|
||||
|
||||
w4a4_config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
kernel = EmulationMxfp4LinearKernel(w4a4_config)
|
||||
assert kernel.quant_dequant_func is quant_dequant_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_emulation_kernel_dequantizes_at_load_and_keeps_activation_qdq(
|
||||
dtype, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
kernel = EmulationMxfp4LinearKernel(
|
||||
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
)
|
||||
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
|
||||
layer = _make_emulation_layer()
|
||||
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
|
||||
x = torch.tensor([[2.0, 4.0]], dtype=dtype)
|
||||
|
||||
with (
|
||||
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", True),
|
||||
patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
|
||||
return_value=dequantized,
|
||||
) as dequant,
|
||||
):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
actual = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = torch.nn.functional.linear(x * 0.5, dequantized.to(dtype))
|
||||
torch.testing.assert_close(actual, expected)
|
||||
dequant.assert_called_once()
|
||||
kernel.quant_dequant_func.assert_called_once_with(x)
|
||||
assert layer.weight.dtype == torch.bfloat16
|
||||
assert layer.weight.device == dequantized.device
|
||||
assert not layer.weight.requires_grad
|
||||
assert layer.weight_scale is not None
|
||||
assert not layer.weight_scale.requires_grad
|
||||
|
||||
|
||||
def test_emulation_kernel_opt_out_dequantizes_per_invocation(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
kernel = EmulationMxfp4LinearKernel(
|
||||
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
)
|
||||
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
|
||||
layer = _make_emulation_layer()
|
||||
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
|
||||
x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", False),
|
||||
patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
|
||||
return_value=dequantized,
|
||||
) as dequant,
|
||||
):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
dequant.assert_not_called()
|
||||
first = kernel.apply_weights(layer, x)
|
||||
second = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = torch.nn.functional.linear(x * 0.5, dequantized)
|
||||
torch.testing.assert_close(first, expected)
|
||||
torch.testing.assert_close(second, expected)
|
||||
assert dequant.call_count == 2
|
||||
assert kernel.quant_dequant_func.call_count == 2
|
||||
assert layer.weight.dtype == torch.uint8
|
||||
assert layer.weight_scale is not None
|
||||
|
||||
|
||||
def test_aiter_kernel_is_supported_requires_native_mx_support():
|
||||
"""AiterMxfp4LinearKernel must not be selected on platforms without
|
||||
native MX compute, even if AITER itself is importable."""
|
||||
with patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform.supports_mx",
|
||||
return_value=False,
|
||||
):
|
||||
is_supported, reason = AiterMxfp4LinearKernel.is_supported()
|
||||
assert not is_supported
|
||||
assert reason
|
||||
|
||||
|
||||
class OOTMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_dispatches_to_registered_kernel(platform_mock):
|
||||
"""init_mxfp4_linear_kernel should select a registered kernel that
|
||||
reports itself as supported, and construct it with a fresh config."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTMxFp4LinearKernel, PlatformEnum.OOT, "mxfp4")
|
||||
|
||||
kernel = init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic)
|
||||
|
||||
assert isinstance(kernel, OOTMxFp4LinearKernel)
|
||||
assert kernel.config == MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
|
||||
|
||||
class UnsupportedMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return False, "never supported"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_raises_when_no_kernel_matches(platform_mock):
|
||||
platform_mock._enum = PlatformEnum.UNSPECIFIED
|
||||
register_linear_kernel(
|
||||
UnsupportedMxFp4LinearKernel, PlatformEnum.UNSPECIFIED, "mxfp4"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to find a kernel"):
|
||||
init_mxfp4_linear_kernel()
|
||||
@@ -1,304 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Copyright (C) 2025 Roberto L. Castro (Roberto.LopezCastro@ist.ac.at).
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.transform.utils.hadamard import deterministic_hadamard_matrix
|
||||
|
||||
from vllm._custom_ops import fusedQuantizeMx, matmul_mxf4_bf16_tn
|
||||
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for these tests.", allow_module_level=True)
|
||||
|
||||
if not (
|
||||
current_platform.has_device_capability(100)
|
||||
or current_platform.has_device_capability(120)
|
||||
):
|
||||
pytest.skip(
|
||||
reason="Tests require compute capability 10.0 (100) or 12.0 (120).",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
# ----- Helpers -----
|
||||
def get_hadamard_matrix(group_size: int, dtype: torch.dtype, device: torch.device):
|
||||
return (
|
||||
deterministic_hadamard_matrix(group_size, dtype=dtype, device=device)
|
||||
* group_size**-0.5
|
||||
)
|
||||
|
||||
|
||||
def _rtne_fp4(x: torch.Tensor):
|
||||
device = x.device
|
||||
grid = torch.tensor(
|
||||
[
|
||||
-6.0,
|
||||
-4.0,
|
||||
-3.0,
|
||||
-2.0,
|
||||
-1.5,
|
||||
-1.0,
|
||||
-0.5,
|
||||
-0.0,
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
],
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
grid_int = torch.tensor(
|
||||
[-1, -2, -3, -4, -5, -6, -7, -8, 0, 1, 2, 3, 4, 5, 6, 7],
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
inds = torch.bucketize(x, grid)
|
||||
lo, hi = (inds - 1).clamp(min=0, max=15), inds.clamp(min=0, max=15)
|
||||
g_lo, g_hi = grid[lo], grid[hi]
|
||||
pick_hi = (g_hi - x < x - g_lo) | (g_hi - x == x - g_lo) & (grid_int[hi] % 2 == 0)
|
||||
y = torch.where(pick_hi, g_hi, g_lo)
|
||||
y_int = torch.where(pick_hi, grid_int[hi], grid_int[lo])
|
||||
y_int_packed = (y_int[..., 1::2] & 0xF) << 4 | y_int[..., ::2] & 0xF
|
||||
return y, y_int_packed
|
||||
|
||||
|
||||
def _dq_fp4(x_e2m1: torch.Tensor, x_e8m0: torch.Tensor, alpha: float):
|
||||
device = x_e2m1.device
|
||||
|
||||
x_e2m1_i32 = x_e2m1.view(dtype=torch.uint8).to(dtype=torch.int32)
|
||||
x_e2m1_unpacked = torch.stack(
|
||||
[x_e2m1_i32 & 0xF, (x_e2m1_i32 >> 4) & 0xF], dim=-1
|
||||
).flatten(start_dim=-2)
|
||||
|
||||
grid_dq = torch.tensor(
|
||||
[
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
],
|
||||
dtype=torch.float64,
|
||||
device=device,
|
||||
)
|
||||
x_fp4_dq = grid_dq[x_e2m1_unpacked]
|
||||
scales_dq = x_e8m0.to(torch.float64)
|
||||
|
||||
x_dq = (x_fp4_dq.unflatten(dim=-1, sizes=(-1, 32)) * scales_dq[..., None]).flatten(
|
||||
start_dim=-2
|
||||
) / alpha
|
||||
return x_dq, x_fp4_dq, scales_dq
|
||||
|
||||
|
||||
def _unpack_mask(clip_mask: torch.Tensor) -> torch.Tensor:
|
||||
clip_mask_unpacked_dq = torch.zeros(
|
||||
*clip_mask.shape[:-1],
|
||||
clip_mask.size(-1) * 8,
|
||||
dtype=torch.bool,
|
||||
device=clip_mask.device,
|
||||
)
|
||||
for i in range(8):
|
||||
clip_mask_unpacked_dq[..., i::8] = (clip_mask >> i) & 1
|
||||
return clip_mask_unpacked_dq
|
||||
|
||||
|
||||
def _forward_quantize_ref(
|
||||
x: torch.Tensor, h: torch.Tensor, rot_size: int, quest: bool = True
|
||||
):
|
||||
device = x.device
|
||||
xh_ref64 = (
|
||||
x.unflatten(dim=-1, sizes=(-1, rot_size)).to(dtype=torch.float64)
|
||||
@ h.reshape(rot_size, rot_size).to(dtype=torch.float64)
|
||||
).flatten(start_dim=-2)
|
||||
|
||||
if quest:
|
||||
scales_ref64_ = (
|
||||
xh_ref64.unflatten(dim=-1, sizes=(-1, 32)).std(dim=-1, correction=0)
|
||||
* (2.92247856 / 6.0)
|
||||
+ 1e-8
|
||||
)
|
||||
else:
|
||||
abs_max = xh_ref64.unflatten(dim=-1, sizes=(-1, 32)).abs().amax(dim=-1)
|
||||
scales_ref64_ = abs_max + 1e-8
|
||||
|
||||
xh_e8m0_ref = scales_ref64_.log2().floor().exp2().to(dtype=torch.float8_e8m0fnu)
|
||||
scales_ref64 = xh_e8m0_ref.to(dtype=torch.float64)
|
||||
|
||||
xh_scaled_ref64 = (
|
||||
xh_ref64.unflatten(dim=-1, sizes=(-1, 32)) / scales_ref64[..., None]
|
||||
).flatten(start_dim=-2)
|
||||
if not quest:
|
||||
xh_scaled_ref64 *= 3
|
||||
|
||||
clip_mask_unpacked_ref = xh_scaled_ref64.abs() < 6.0
|
||||
clip_mask_ref = torch.zeros(
|
||||
*x.shape[:-1], x.size(-1) // 8, dtype=torch.uint8, device=device
|
||||
)
|
||||
for i in range(8):
|
||||
clip_mask_ref |= clip_mask_unpacked_ref[..., i::8].to(dtype=torch.uint8) << i
|
||||
|
||||
xh_fp4_ref, xh_e2m1_ref = _rtne_fp4(xh_scaled_ref64)
|
||||
xh_dq, xh_fp4_dq, scales_dq = _dq_fp4(
|
||||
xh_e2m1_ref, xh_e8m0_ref, alpha=1.0 if quest else 3.0
|
||||
)
|
||||
clip_mask_unpacked_dq = _unpack_mask(clip_mask_ref)
|
||||
|
||||
assert xh_fp4_dq.equal(xh_fp4_ref)
|
||||
assert scales_dq.equal(scales_ref64)
|
||||
assert clip_mask_unpacked_dq.equal(clip_mask_unpacked_ref)
|
||||
|
||||
return (
|
||||
xh_dq,
|
||||
clip_mask_unpacked_ref,
|
||||
(xh_e2m1_ref, xh_e8m0_ref, clip_mask_ref),
|
||||
)
|
||||
|
||||
|
||||
DTYPE = torch.bfloat16
|
||||
DEVICE = torch.device("cuda:0")
|
||||
|
||||
ROT_SIZES = [32, 64, 128]
|
||||
SEEDS = [0]
|
||||
BATCHES = [1, 16]
|
||||
|
||||
LLAMA_MODELS = {
|
||||
"7B": [(4096, 3 * 4096), (4096, 4096), (4096, 2 * 10752), (10752, 4096)],
|
||||
"13B": [(5120, 3 * 5120), (5120, 5120), (5120, 2 * 13568), (13568, 5120)],
|
||||
"33B": [(6656, 3 * 6656), (6656, 6656), (6656, 2 * 17664), (17664, 6656)],
|
||||
"70B": [(8192, 3 * 8192), (8192, 8192), (8192, 2 * 21760), (21760, 8192)],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _seed_each_test():
|
||||
set_random_seed(0)
|
||||
np.random.seed(0)
|
||||
torch.random.manual_seed(0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rot_size", ROT_SIZES)
|
||||
@torch.inference_mode()
|
||||
def test_fused_quantization_absmax(rot_size: int):
|
||||
dtype, device = DTYPE, DEVICE
|
||||
h = get_hadamard_matrix(rot_size, dtype, device)
|
||||
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
|
||||
|
||||
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size, quest=False)
|
||||
xh_e2m1, xh_e8m0 = fusedQuantizeMx(x, h, method="abs_max")
|
||||
xh_e8m0 = xh_e8m0.reshape(2, 4096, 4096 // 32)
|
||||
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e8m0, alpha=3.0)
|
||||
|
||||
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
|
||||
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-4
|
||||
|
||||
m, n, k = 1, 504, 4096
|
||||
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
|
||||
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
|
||||
|
||||
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="abs_max")
|
||||
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="abs_max")
|
||||
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
|
||||
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
|
||||
out_ref = a_dq @ b_dq.transpose(-2, -1)
|
||||
|
||||
a_scale_block = to_blocked(a_e8m0, backend="triton")
|
||||
b_scale_block = to_blocked(b_e8m0, backend="triton")
|
||||
alpha = torch.tensor([1.0], device=device)
|
||||
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
|
||||
assert out.equal(out_ref.to(dtype=out.dtype))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rot_size", ROT_SIZES)
|
||||
@torch.inference_mode()
|
||||
def test_fused_quantization_quest(rot_size: int):
|
||||
dtype, device = DTYPE, DEVICE
|
||||
h = get_hadamard_matrix(rot_size, dtype, device)
|
||||
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
|
||||
|
||||
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size, quest=True)
|
||||
xh_e2m1, xh_e8m0 = fusedQuantizeMx(x, h, method="quest")
|
||||
xh_e8m0 = xh_e8m0.reshape(2, 4096, 4096 // 32)
|
||||
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e8m0, alpha=1.0)
|
||||
|
||||
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
|
||||
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-4
|
||||
|
||||
m, n, k = 504, 504, 2048
|
||||
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
|
||||
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
|
||||
|
||||
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="quest")
|
||||
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="quest")
|
||||
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
|
||||
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
|
||||
out_ref = a_dq @ b_dq.transpose(-2, -1)
|
||||
|
||||
a_scale_block = to_blocked(a_e8m0, backend="triton")
|
||||
b_scale_block = to_blocked(b_e8m0, backend="triton")
|
||||
alpha = torch.tensor([1.0], device=device)
|
||||
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
|
||||
assert out.equal(out_ref.to(dtype=out.dtype))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", list(LLAMA_MODELS.keys()))
|
||||
@pytest.mark.parametrize("layer_idx", [0, 1, 2, 3])
|
||||
@pytest.mark.parametrize("batch", [1, 16])
|
||||
@pytest.mark.parametrize("had_size", ROT_SIZES)
|
||||
@torch.inference_mode()
|
||||
def test_llama_shapes(model: str, layer_idx: int, batch: int, had_size: int):
|
||||
dtype, device = DTYPE, DEVICE
|
||||
m = batch
|
||||
k, n = LLAMA_MODELS[model][layer_idx]
|
||||
|
||||
h = get_hadamard_matrix(had_size, dtype, device)
|
||||
|
||||
a = torch.rand(m, k, dtype=dtype, device=device) * 25.0
|
||||
b = torch.rand(n, k, dtype=dtype, device=device) * 25.0
|
||||
|
||||
a_e2m1, a_e8m0 = fusedQuantizeMx(a, h, method="quest")
|
||||
b_e2m1, b_e8m0 = fusedQuantizeMx(b, h, method="quest")
|
||||
|
||||
a_dq, *_ = _dq_fp4(a_e2m1, a_e8m0[:m, :k], alpha=1.0)
|
||||
b_dq, *_ = _dq_fp4(b_e2m1, b_e8m0[:n, :k], alpha=1.0)
|
||||
out_ref = a_dq @ b_dq.transpose(-2, -1)
|
||||
|
||||
a_scale_block = to_blocked(a_e8m0, backend="triton")
|
||||
b_scale_block = to_blocked(b_e8m0, backend="triton")
|
||||
alpha = torch.tensor([1.0], device=device)
|
||||
out = matmul_mxf4_bf16_tn(a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha)
|
||||
assert out.equal(out_ref.to(dtype=out.dtype))
|
||||
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests that triton_kernel_moe_forward correctly applies expert_map
|
||||
remapping when expert parallelism (EP) is enabled.
|
||||
|
||||
When expert_map is provided, global expert IDs are remapped to local IDs
|
||||
via topk + expert_map remap + make_routing_data before building routing
|
||||
structures, and the expert_map passed downstream to triton_kernel_fused_experts
|
||||
is None (already applied).
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class TestTritonMoeForwardExpertMap:
|
||||
"""Test that triton_kernel_moe_forward applies expert_map remapping
|
||||
when expert_map is provided (EP active)."""
|
||||
|
||||
def test_expert_map_remap(self):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
mock_expert_map = torch.tensor([0, -1, 1, -1], device=device)
|
||||
|
||||
from vllm.utils.import_utils import import_triton_kernels
|
||||
|
||||
import_triton_kernels()
|
||||
|
||||
mock_routing_data = MagicMock()
|
||||
mock_gather = MagicMock()
|
||||
mock_scatter = MagicMock()
|
||||
|
||||
with (
|
||||
patch("triton_kernels.topk.topk") as mock_topk,
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"gpt_oss_triton_kernels_moe.make_routing_data"
|
||||
) as mock_make_routing,
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"gpt_oss_triton_kernels_moe.triton_kernel_fused_experts"
|
||||
) as mock_fused_experts,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import ( # noqa: E501
|
||||
triton_kernel_moe_forward,
|
||||
)
|
||||
|
||||
sparse_result = MagicMock()
|
||||
sparse_result.indx = torch.tensor([[0, 2]], dtype=torch.int32)
|
||||
sparse_result.vals = torch.tensor([[0.6, 0.4]])
|
||||
mock_topk.return_value = sparse_result
|
||||
|
||||
mock_make_routing.return_value = (
|
||||
mock_routing_data,
|
||||
mock_gather,
|
||||
mock_scatter,
|
||||
)
|
||||
|
||||
mock_fused_experts.return_value = torch.zeros((1, 8), device=device)
|
||||
|
||||
hidden = torch.randn((1, 8), device=device)
|
||||
w1 = torch.randn((2, 8, 16), device=device)
|
||||
w2 = torch.randn((2, 8, 8), device=device)
|
||||
logits = torch.randn((1, 4), device=device)
|
||||
|
||||
triton_kernel_moe_forward(
|
||||
hidden_states=hidden,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
gating_output=logits,
|
||||
topk=2,
|
||||
renormalize=True,
|
||||
expert_map=mock_expert_map,
|
||||
)
|
||||
|
||||
mock_topk.assert_called_once()
|
||||
mock_make_routing.assert_called_once()
|
||||
|
||||
# expert_map should be None in the fused_experts call
|
||||
# (already applied).
|
||||
call_kwargs = mock_fused_experts.call_args
|
||||
assert call_kwargs[1].get("expert_map") is None or (len(call_kwargs[0]) > 0)
|
||||
@@ -1,150 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MXFP6 linear kernel selection logic (CPU-only).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_mxfp6_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
EmulationMxfp6LinearKernel,
|
||||
MxFp6LinearKernel,
|
||||
MxFp6LinearLayerConfig,
|
||||
init_mxfp6_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Dynamic,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# The only implementation available at the moment is software emulation.
|
||||
_WEIGHT_QUANT_KEYS = [kMxfp6E3M2Static, kMxfp6E2M3Static]
|
||||
|
||||
|
||||
def test_can_implement_is_abstract():
|
||||
"""Test that can_implement()/is_supported() are properly defined."""
|
||||
assert hasattr(MxFp6LinearKernel, "can_implement")
|
||||
assert hasattr(MxFp6LinearKernel, "is_supported")
|
||||
|
||||
|
||||
def test_emulation_kernel_rejects_non_mxfp6_weights():
|
||||
"""EmulationMxfp6LinearKernel must not implement a non-MXFP6 weight
|
||||
format."""
|
||||
config = MxFp6LinearLayerConfig(weight_quant_key=kMxfp4Static)
|
||||
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("weight_quant_key", _WEIGHT_QUANT_KEYS)
|
||||
@pytest.mark.parametrize(
|
||||
"activation_quant_key",
|
||||
[None, kMxfp4Dynamic, kMxfp6E3M2Dynamic, kMxfp6E2M3Dynamic],
|
||||
)
|
||||
def test_emulation_kernel_accepts_any_supported_config(
|
||||
weight_quant_key, activation_quant_key
|
||||
):
|
||||
"""EmulationMxfp6LinearKernel is the only backend today: it must accept
|
||||
every supported weight/activation format combination."""
|
||||
config = MxFp6LinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key
|
||||
)
|
||||
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
|
||||
assert can_implement, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("weight_quant_key", _WEIGHT_QUANT_KEYS)
|
||||
def test_emulation_kernel_rejects_non_mxfp4_or_mxfp6_activation(weight_quant_key):
|
||||
config = MxFp6LinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key, activation_quant_key=kMxfp4Static
|
||||
)
|
||||
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
class OOTMxFp6LinearKernel(MxFp6LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp6_linear_kernel_dispatches_to_registered_kernel(platform_mock):
|
||||
"""init_mxfp6_linear_kernel should select a registered kernel that
|
||||
reports itself as supported/able to implement the given config, and
|
||||
construct it with that exact config."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTMxFp6LinearKernel, PlatformEnum.OOT, "mxfp6")
|
||||
|
||||
kernel = init_mxfp6_linear_kernel(
|
||||
weight_quant_key=kMxfp6E3M2Static, activation_quant_key=kMxfp6E3M2Dynamic
|
||||
)
|
||||
|
||||
assert isinstance(kernel, OOTMxFp6LinearKernel)
|
||||
assert kernel.config == MxFp6LinearLayerConfig(
|
||||
weight_quant_key=kMxfp6E3M2Static, activation_quant_key=kMxfp6E3M2Dynamic
|
||||
)
|
||||
|
||||
|
||||
class UnsupportedMxFp6LinearKernel(MxFp6LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return False, "never supported"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp6LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp6_linear_kernel_raises_when_no_kernel_matches(platform_mock):
|
||||
platform_mock._enum = PlatformEnum.UNSPECIFIED
|
||||
register_linear_kernel(
|
||||
UnsupportedMxFp6LinearKernel, PlatformEnum.UNSPECIFIED, "mxfp6"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to find a kernel"):
|
||||
init_mxfp6_linear_kernel(weight_quant_key=kMxfp6E3M2Static)
|
||||
@@ -1,135 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MXFP8 linear kernel selection logic (CPU-only)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_mxfp8_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
MarlinMxfp8LinearKernel,
|
||||
Mxfp8LinearLayerConfig,
|
||||
init_mxfp8_linear_kernel,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# Kernels backed by FlashInfer mm_mxfp8, which requires N, K >= 128 and
|
||||
# K % 32 == 0.
|
||||
_MM_MXFP8_KERNELS = [
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
]
|
||||
|
||||
_SUPPORTED_SHAPE = (4096, 4096)
|
||||
# (N, K): N < 128, K < 128, K % 32 != 0.
|
||||
_UNSUPPORTED_SHAPES = [(64, 4096), (4096, 64), (4096, 4112)]
|
||||
|
||||
# (N, K) with N < 128 or K < 128, which only Marlin runs.
|
||||
_MARLIN_ONLY_SHAPES = [(32, 256), (64, 256), (96, 256), (256, 64)]
|
||||
# (N, K): N <= 0, K <= 0, K % 32 != 0.
|
||||
_MARLIN_UNSUPPORTED_SHAPES = [(0, 256), (256, 0), (256, 100)]
|
||||
|
||||
_SELECTION_CASES = [
|
||||
(100, (128, 128), FlashInferCutedslMxfp8LinearKernel),
|
||||
(100, (130, 256), FlashInferCutedslMxfp8LinearKernel),
|
||||
(103, (128, 128), FlashInferCutedslMxfp8LinearKernel),
|
||||
(103, (130, 256), FlashInferCutedslMxfp8LinearKernel),
|
||||
(120, (128, 128), FlashInferCutlassMxfp8LinearKernel),
|
||||
(120, (130, 256), MarlinMxfp8LinearKernel),
|
||||
(121, (128, 128), FlashInferCutlassMxfp8LinearKernel),
|
||||
(121, (130, 256), MarlinMxfp8LinearKernel),
|
||||
*[
|
||||
(capability, shape, MarlinMxfp8LinearKernel)
|
||||
for capability in (100, 103, 120, 121)
|
||||
for shape in _MARLIN_ONLY_SHAPES
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS)
|
||||
def test_mm_mxfp8_kernels_accept_supported_shape(kernel_cls):
|
||||
config = Mxfp8LinearLayerConfig(weight_shape=_SUPPORTED_SHAPE)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert can_implement, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kernel_cls", _MM_MXFP8_KERNELS)
|
||||
@pytest.mark.parametrize("weight_shape", _UNSUPPORTED_SHAPES)
|
||||
def test_mm_mxfp8_kernels_reject_unsupported_shape(kernel_cls, weight_shape):
|
||||
config = Mxfp8LinearLayerConfig(weight_shape=weight_shape)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("weight_shape", _MARLIN_UNSUPPORTED_SHAPES)
|
||||
def test_marlin_mxfp8_rejects_unsupported_shape(weight_shape):
|
||||
config = Mxfp8LinearLayerConfig(weight_shape=weight_shape)
|
||||
can_implement, reason = MarlinMxfp8LinearKernel.can_implement(config)
|
||||
assert not can_implement
|
||||
assert reason
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _patch_cuda_platform(capability: int):
|
||||
"""Patch the platform so kernel selection runs as on a CUDA GPU with the
|
||||
given compute capability. The FlashInfer and Marlin kernels report as
|
||||
supported, CuTe-DSL only on SM100/103."""
|
||||
with (
|
||||
patch("vllm.model_executor.kernels.linear.current_platform") as platform,
|
||||
patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp8.flashinfer.current_platform."
|
||||
"is_device_capability_family",
|
||||
side_effect=lambda cap, device_id=0: cap // 10 == capability // 10,
|
||||
),
|
||||
patch.object(
|
||||
FlashInferCutedslMxfp8LinearKernel,
|
||||
"is_supported",
|
||||
return_value=(capability in (100, 103), None),
|
||||
),
|
||||
patch.object(
|
||||
FlashInferCutlassMxfp8LinearKernel,
|
||||
"is_supported",
|
||||
return_value=(True, None),
|
||||
),
|
||||
patch.object(
|
||||
MarlinMxfp8LinearKernel, "is_supported", return_value=(True, None)
|
||||
),
|
||||
):
|
||||
platform._enum = PlatformEnum.CUDA
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("capability", "expected"),
|
||||
[(100, True), (103, True), (120, False), (121, False)],
|
||||
)
|
||||
def test_cutlass_mxfp8_requires_aligned_n_only_on_sm12x(capability, expected):
|
||||
"""(130, 256) passes the generic mm_mxfp8 limits. Only the SM12x CUTLASS
|
||||
backend needs N % 32 == 0, so SM100/103 must keep accepting it."""
|
||||
config = Mxfp8LinearLayerConfig(weight_shape=(130, 256))
|
||||
with _patch_cuda_platform(capability):
|
||||
can_implement, _ = FlashInferCutlassMxfp8LinearKernel.can_implement(config)
|
||||
assert can_implement == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("capability", "weight_shape", "expected_kernel_cls"), _SELECTION_CASES
|
||||
)
|
||||
def test_init_mxfp8_linear_kernel(capability, weight_shape, expected_kernel_cls):
|
||||
"""A layer that mm_mxfp8 cannot handle must fall through to the next
|
||||
kernel in the CUDA priority list instead of being selected and failing
|
||||
in apply_weights."""
|
||||
with _patch_cuda_platform(capability):
|
||||
kernel = init_mxfp8_linear_kernel(weight_shape=weight_shape)
|
||||
|
||||
assert isinstance(kernel, expected_kernel_cls)
|
||||
assert kernel.config == Mxfp8LinearLayerConfig(weight_shape=weight_shape)
|
||||
@@ -1,866 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
break_fp4_bytes,
|
||||
kE2M1ToFloat,
|
||||
)
|
||||
from safetensors import safe_open
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import KernelConfig
|
||||
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
|
||||
from vllm.model_executor.kernels.linear.nvfp4.emulation import (
|
||||
EmulationNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import (
|
||||
Nvfp4QuantizationEmulationTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.model_executor.layers.quantization.utils import (
|
||||
nvfp4_emulation_utils,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
ref_nvfp4_quant_dequant,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.triton_utils import triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
else:
|
||||
|
||||
def on_gfx950() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
MOE_MODEL_CONFIGS = {
|
||||
"nvidia/Qwen3-30B-A3B-NVFP4": {
|
||||
"shards": ["model-00001-of-00004.safetensors"],
|
||||
"expert_prefix": "model.layers.9.mlp.experts.",
|
||||
# Position of the expert index in the dot-split key.
|
||||
"expert_idx_pos": 5,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def loaded_model_files():
|
||||
return {
|
||||
model_id: hf_api().snapshot_download(
|
||||
repo_id=model_id, allow_patterns=config["shards"]
|
||||
)
|
||||
for model_id, config in MOE_MODEL_CONFIGS.items()
|
||||
}
|
||||
|
||||
|
||||
class Nvfp4QuantizationEmulationTritonExpertsReference(TritonExperts):
|
||||
"""Extension of TritonExperts to support emulated NVFP4 MoE experts.
|
||||
|
||||
It may be used for NVFP4 models when the device does not have
|
||||
native support for this dtype.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
|
||||
# `TritonExperts.apply` expects pre-dequantized weights,
|
||||
# which we handle in `apply` below.
|
||||
self.w1_scale_val = self.quant_config.w1_scale
|
||||
self.w2_scale_val = self.quant_config.w2_scale
|
||||
|
||||
self.quant_config._w1.scale = None
|
||||
self.quant_config._w2.scale = None
|
||||
|
||||
self.quantization_emulation = True
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return "nvfp4"
|
||||
|
||||
@property
|
||||
def a1_scale(self) -> torch.Tensor | None:
|
||||
return self.quant_config.a1_gscale
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (kNvfp4Static, kNvfp4Dynamic)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert w1.dtype == torch.uint8
|
||||
assert w2.dtype == torch.uint8
|
||||
|
||||
# Dequantize w1 from packed NVFP4 to fp16/bf16
|
||||
w13_global_scale = self.quant_config.g1_alphas
|
||||
|
||||
w1_dequant = dequantize_to_dtype(
|
||||
tensor_fp4=w1,
|
||||
tensor_sf=self.w1_scale_val,
|
||||
global_scale=w13_global_scale,
|
||||
dtype=hidden_states.dtype,
|
||||
block_size=16,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
# Dequantize w2 from packed NVFP4 to fp16/bf16
|
||||
w2_global_scale = self.quant_config.g2_alphas
|
||||
|
||||
w2_dequant = dequantize_to_dtype(
|
||||
tensor_fp4=w2,
|
||||
tensor_sf=self.w2_scale_val,
|
||||
global_scale=w2_global_scale,
|
||||
dtype=hidden_states.dtype,
|
||||
block_size=16,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
super().apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=w1_dequant,
|
||||
w2=w2_dequant,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=self.quant_config.a2_gscale,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config_kwargs", "expected_reason"),
|
||||
[
|
||||
({"has_bias": True}, "kernel does not support bias"),
|
||||
({"is_lora_enabled": True}, "kernel does not support LoRA"),
|
||||
],
|
||||
)
|
||||
def test_nvfp4_emulation_support_check_rejects_bias_and_lora(
|
||||
config_kwargs: dict[str, bool],
|
||||
expected_reason: str,
|
||||
) -> None:
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=2,
|
||||
experts_per_token=1,
|
||||
hidden_dim=16,
|
||||
intermediate_size=16,
|
||||
num_local_experts=2,
|
||||
num_logical_experts=2,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
**config_kwargs,
|
||||
)
|
||||
|
||||
supported, reason = Nvfp4QuantizationEmulationTritonExperts.is_supported_config(
|
||||
Nvfp4QuantizationEmulationTritonExperts,
|
||||
moe_config,
|
||||
kNvfp4Static,
|
||||
kNvfp4Dynamic,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert not supported
|
||||
assert reason == expected_reason
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Triton NVFP4 kernel requires CUDA.",
|
||||
)
|
||||
def test_triton_dequantize_nvfp4(monkeypatch, loaded_model_files) -> None:
|
||||
"""Test the Triton dequantization kernel against the CPU reference
|
||||
using real NVFP4 weights from a checkpoint.
|
||||
|
||||
Tests both 2D (attention projection) and 3D (stacked MoE experts).
|
||||
"""
|
||||
checkpoint_path = loaded_model_files["nvidia/Qwen3-30B-A3B-NVFP4"]
|
||||
shards = cast(list[str], MOE_MODEL_CONFIGS["nvidia/Qwen3-30B-A3B-NVFP4"]["shards"])
|
||||
shard_path = f"{checkpoint_path}/{shards[0]}"
|
||||
block_size = 16
|
||||
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
all_keys = list(f.keys())
|
||||
|
||||
# 2D case: attention projection
|
||||
tensor_fp4_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight")
|
||||
tensor_sf_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale")
|
||||
global_scale_2d = f.get_tensor("model.layers.9.self_attn.k_proj.weight_scale_2")
|
||||
|
||||
# 3D case: stack ALL experts for layer 9 up_proj
|
||||
expert_prefix = "model.layers.9.mlp.experts."
|
||||
expert_indices = sorted(
|
||||
int(key.split(".")[5])
|
||||
for key in all_keys
|
||||
if key.startswith(expert_prefix) and key.endswith(".up_proj.weight")
|
||||
)
|
||||
assert len(expert_indices) > 0
|
||||
|
||||
all_fp4 = []
|
||||
all_sf = []
|
||||
all_global_scale = []
|
||||
for index in expert_indices:
|
||||
name = f"{expert_prefix}{index}.up_proj"
|
||||
all_fp4.append(f.get_tensor(f"{name}.weight"))
|
||||
all_sf.append(f.get_tensor(f"{name}.weight_scale"))
|
||||
all_global_scale.append(f.get_tensor(f"{name}.weight_scale_2"))
|
||||
|
||||
tensor_fp4_3d = torch.stack(all_fp4)
|
||||
tensor_sf_3d = torch.stack(all_sf)
|
||||
global_scale_3d = torch.stack(all_global_scale)
|
||||
|
||||
test_cases = [
|
||||
("2D base", tensor_fp4_2d, tensor_sf_2d, global_scale_2d),
|
||||
(
|
||||
"2D 2x rows",
|
||||
tensor_fp4_2d.repeat(2, 1),
|
||||
tensor_sf_2d.repeat(2, 1),
|
||||
global_scale_2d,
|
||||
),
|
||||
(
|
||||
"2D 4x rows",
|
||||
tensor_fp4_2d.repeat(4, 1),
|
||||
tensor_sf_2d.repeat(4, 1),
|
||||
global_scale_2d,
|
||||
),
|
||||
(
|
||||
"2D 2x cols",
|
||||
tensor_fp4_2d.repeat(1, 2),
|
||||
tensor_sf_2d.repeat(1, 2),
|
||||
global_scale_2d,
|
||||
),
|
||||
("3D base", tensor_fp4_3d, tensor_sf_3d, global_scale_3d),
|
||||
(
|
||||
"3D 2x experts",
|
||||
tensor_fp4_3d.repeat(2, 1, 1),
|
||||
tensor_sf_3d.repeat(2, 1, 1),
|
||||
global_scale_3d.repeat(2),
|
||||
),
|
||||
(
|
||||
"3D 2x rows",
|
||||
tensor_fp4_3d.repeat(1, 2, 1),
|
||||
tensor_sf_3d.repeat(1, 2, 1),
|
||||
global_scale_3d,
|
||||
),
|
||||
(
|
||||
"3D 2x cols",
|
||||
tensor_fp4_3d.repeat(1, 1, 2),
|
||||
tensor_sf_3d.repeat(1, 1, 2),
|
||||
global_scale_3d,
|
||||
),
|
||||
]
|
||||
|
||||
quantiles = [0.5, 0.001, 0.999]
|
||||
|
||||
# Move the E2M1 lookup table to CUDA ahead of time, as would normally
|
||||
# happen during model loading (process_weights_after_loading). Both the
|
||||
# Triton and PyTorch reference paths run on CUDA.
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val = (
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda()
|
||||
)
|
||||
|
||||
for label, tensor_fp4, tensor_sf, global_scale in test_cases:
|
||||
fp4_cuda = tensor_fp4.cuda()
|
||||
sf_cuda = tensor_sf.cuda()
|
||||
gs_cuda = global_scale.cuda()
|
||||
|
||||
# Triton path
|
||||
triton_result = dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
sf_cuda,
|
||||
gs_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
# Reference path (PyTorch ops on CUDA, Triton dispatch disabled)
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
reference = dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
sf_cuda,
|
||||
gs_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_result, reference, atol=0, rtol=0)
|
||||
|
||||
# Benchmark
|
||||
shape = list(tensor_fp4.shape)
|
||||
|
||||
def _triton_bench(
|
||||
fp4_cuda=fp4_cuda,
|
||||
scale_cuda=sf_cuda,
|
||||
global_scale_cuda=gs_cuda,
|
||||
block_size=block_size,
|
||||
):
|
||||
return dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
scale_cuda,
|
||||
global_scale_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
triton_ms, triton_min, triton_max = triton.testing.do_bench(
|
||||
_triton_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
def _reference_bench(
|
||||
fp4_cuda=fp4_cuda,
|
||||
scale_cuda=sf_cuda,
|
||||
global_scale_cuda=gs_cuda,
|
||||
block_size=block_size,
|
||||
):
|
||||
with monkeypatch.context() as m2:
|
||||
m2.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
dequantize_to_dtype(
|
||||
fp4_cuda,
|
||||
scale_cuda,
|
||||
global_scale_cuda,
|
||||
torch.bfloat16,
|
||||
block_size,
|
||||
swizzle=False,
|
||||
)
|
||||
|
||||
ref_ms, ref_min, ref_max = triton.testing.do_bench(
|
||||
_reference_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
|
||||
print(f" dequantize {label} {shape}:")
|
||||
print(
|
||||
f" triton: median={triton_ms:.3f}ms, "
|
||||
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
|
||||
)
|
||||
print(
|
||||
f" reference: median={ref_ms:.3f}ms, "
|
||||
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
|
||||
)
|
||||
print(f" speedup: {speedup:.2f}x")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Triton NVFP4 kernel requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"m, k",
|
||||
[
|
||||
(1, 16),
|
||||
(1, 4096),
|
||||
(2, 4096),
|
||||
(4, 4096),
|
||||
(8, 4096),
|
||||
(16, 4096),
|
||||
(24, 4096),
|
||||
(32, 4096),
|
||||
(1, 8192),
|
||||
(2, 8192),
|
||||
(4, 8192),
|
||||
(8, 8192),
|
||||
(16, 8192),
|
||||
(24, 8192),
|
||||
(32, 8192),
|
||||
(1, 32),
|
||||
(2, 48),
|
||||
(7, 64),
|
||||
(16, 128),
|
||||
(33, 160),
|
||||
(128, 256),
|
||||
(256, 512),
|
||||
(1024, 1024),
|
||||
(5120, 2048),
|
||||
(2048, 4096),
|
||||
(4096, 7168),
|
||||
(8192, 8192),
|
||||
(128, 16384),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("global_scale_value", [0.5, 1.0, 0.001])
|
||||
def test_triton_nvfp4_quant_dequant(
|
||||
monkeypatch, m: int, k: int, global_scale_value: float
|
||||
) -> None:
|
||||
"""Test the Triton quant-dequant kernel against the CPU reference."""
|
||||
block_size = 16
|
||||
x = torch.randn(m, k, dtype=torch.bfloat16, device="cuda")
|
||||
global_scale = torch.tensor(global_scale_value, dtype=torch.float32, device="cuda")
|
||||
|
||||
# Triton path
|
||||
triton_result = ref_nvfp4_quant_dequant(x, global_scale, block_size)
|
||||
|
||||
# CPU reference path
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
reference = ref_nvfp4_quant_dequant(x.cpu(), global_scale.cpu(), block_size)
|
||||
|
||||
torch.testing.assert_close(triton_result.cpu(), reference, atol=0, rtol=0)
|
||||
|
||||
# Benchmark (both paths on CUDA tensors for fair comparison)
|
||||
quantiles = [0.5, 0.001, 0.999]
|
||||
|
||||
def _triton_bench(
|
||||
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
|
||||
):
|
||||
return ref_nvfp4_quant_dequant(
|
||||
input_tensor, input_global_scale, input_block_size
|
||||
)
|
||||
|
||||
triton_ms, triton_min, triton_max = triton.testing.do_bench(
|
||||
_triton_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
def _reference_bench(
|
||||
input_tensor=x, input_global_scale=global_scale, input_block_size=block_size
|
||||
):
|
||||
with monkeypatch.context() as mp2:
|
||||
mp2.setattr(
|
||||
nvfp4_emulation_utils.current_platform,
|
||||
"is_cuda_alike",
|
||||
lambda: False,
|
||||
)
|
||||
ref_nvfp4_quant_dequant(input_tensor, input_global_scale, input_block_size)
|
||||
|
||||
ref_ms, ref_min, ref_max = triton.testing.do_bench(
|
||||
_reference_bench, quantiles=quantiles
|
||||
)
|
||||
|
||||
speedup = ref_ms / triton_ms if triton_ms > 0 else float("inf")
|
||||
print(f" quant_dequant [{m}x{k}] gs={global_scale_value}:")
|
||||
print(
|
||||
f" triton: median={triton_ms:.3f}ms, "
|
||||
f"min={triton_min:.3f}ms, max={triton_max:.3f}ms"
|
||||
)
|
||||
print(
|
||||
f" reference: median={ref_ms:.3f}ms, "
|
||||
f"min={ref_min:.3f}ms, max={ref_max:.3f}ms"
|
||||
)
|
||||
print(f" speedup: {speedup:.2f}x")
|
||||
|
||||
|
||||
def _load_nvfp4_moe_weights(
|
||||
model_files: dict[str, str],
|
||||
model_id: str,
|
||||
tensor_parallel_size: int,
|
||||
max_experts: int | None = None,
|
||||
):
|
||||
"""Load and stack NVFP4 MoE weights from checkpoint shards.
|
||||
|
||||
Returns (w1, w1_scale, w1_gscale, w2, w2_scale, w2_gscale,
|
||||
a1_gscale, a2_gscale, num_experts, hidden_dim,
|
||||
intermediate_size).
|
||||
|
||||
When max_experts is set, only the first max_experts experts are loaded.
|
||||
|
||||
When tensor_parallel_size > 1, the N dimension of w1 and the K
|
||||
dimension of w2 are narrowed to the first TP shard (simulating
|
||||
column-parallel on w1 / row-parallel on w2).
|
||||
"""
|
||||
cfg = MOE_MODEL_CONFIGS[model_id]
|
||||
shards = cast(list[str], cfg["shards"])
|
||||
checkpoint_path = model_files[model_id]
|
||||
|
||||
expert_prefix = cfg["expert_prefix"]
|
||||
idx_pos = cast(int, cfg["expert_idx_pos"])
|
||||
|
||||
# Collect all tensors across shards into a flat dict — an expert's
|
||||
# tensors may be split across multiple shard files.
|
||||
all_tensors: dict[str, torch.Tensor] = {}
|
||||
for shard_name in shards:
|
||||
shard_path = f"{checkpoint_path}/{shard_name}"
|
||||
with safe_open(shard_path, framework="pt", device="cpu") as f:
|
||||
for key in f.keys(): # noqa: SIM118
|
||||
if key.startswith(expert_prefix):
|
||||
all_tensors[key] = f.get_tensor(key)
|
||||
|
||||
expert_indices = sorted(
|
||||
{
|
||||
int(key.split(".")[idx_pos])
|
||||
for key in all_tensors
|
||||
if key.endswith(".gate_proj.weight")
|
||||
}
|
||||
)
|
||||
if max_experts is not None:
|
||||
expert_indices = expert_indices[:max_experts]
|
||||
num_experts = len(expert_indices)
|
||||
|
||||
gate_weights, up_weights, down_weights = [], [], []
|
||||
gate_scales, up_scales, down_scales = [], [], []
|
||||
gate_gscales, up_gscales, down_gscales = [], [], []
|
||||
a1_scales, a2_scales = [], []
|
||||
|
||||
for idx in expert_indices:
|
||||
prefix = f"{expert_prefix}{idx}"
|
||||
gate_weights.append(all_tensors[f"{prefix}.gate_proj.weight"])
|
||||
gate_scales.append(all_tensors[f"{prefix}.gate_proj.weight_scale"])
|
||||
gate_gscales.append(all_tensors[f"{prefix}.gate_proj.weight_scale_2"])
|
||||
up_weights.append(all_tensors[f"{prefix}.up_proj.weight"])
|
||||
up_scales.append(all_tensors[f"{prefix}.up_proj.weight_scale"])
|
||||
up_gscales.append(all_tensors[f"{prefix}.up_proj.weight_scale_2"])
|
||||
down_weights.append(all_tensors[f"{prefix}.down_proj.weight"])
|
||||
down_scales.append(all_tensors[f"{prefix}.down_proj.weight_scale"])
|
||||
down_gscales.append(all_tensors[f"{prefix}.down_proj.weight_scale_2"])
|
||||
a1_scales.append(all_tensors[f"{prefix}.gate_proj.input_scale"])
|
||||
a2_scales.append(all_tensors[f"{prefix}.down_proj.input_scale"])
|
||||
|
||||
# Stack into MoE format.
|
||||
# w1 = [E, 2*intermediate, hidden//2] (gate + up concatenated)
|
||||
w1 = torch.stack(
|
||||
[torch.cat([g, u], dim=0) for g, u in zip(gate_weights, up_weights)]
|
||||
).cuda()
|
||||
w1_scale = torch.stack(
|
||||
[torch.cat([g, u], dim=0) for g, u in zip(gate_scales, up_scales)]
|
||||
).cuda()
|
||||
w1_gscale = torch.stack(gate_gscales).cuda()
|
||||
|
||||
# w2 = [E, hidden, intermediate//2]
|
||||
w2 = torch.stack(down_weights).cuda()
|
||||
w2_scale = torch.stack(down_scales).cuda()
|
||||
w2_gscale = torch.stack(down_gscales).cuda()
|
||||
|
||||
a13_scale_raw = torch.stack(a1_scales).cuda()
|
||||
a2_scale_raw = torch.stack(a2_scales).cuda()
|
||||
|
||||
# Apply EMULATION transforms (matches oracle/nvfp4.py).
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val = (
|
||||
nvfp4_emulation_utils.kE2M1ToFloat_handle.val.cuda()
|
||||
)
|
||||
a1_gscale = 1.0 / a13_scale_raw.max().to(torch.float32)
|
||||
a2_gscale = 1.0 / a2_scale_raw.max().to(torch.float32)
|
||||
|
||||
# ── Simulate TP sharding ──
|
||||
# w1 (gate_up): column-parallel → shard the N dimension (dim 1).
|
||||
# w2 (down): row-parallel → shard the K dimension (dim 2,
|
||||
# which is the packed K//2 dim).
|
||||
# Scales follow the same sharding on the corresponding dimension.
|
||||
tp = tensor_parallel_size
|
||||
if tp > 1:
|
||||
n1 = w1.size(1) // tp
|
||||
w1 = w1[:, :n1, :].contiguous()
|
||||
w1_scale = w1_scale[:, :n1, :].contiguous()
|
||||
|
||||
k2_packed = w2.size(2) // tp
|
||||
k2_scale = w2_scale.size(2) // tp
|
||||
w2 = w2[:, :, :k2_packed].contiguous()
|
||||
w2_scale = w2_scale[:, :, :k2_scale].contiguous()
|
||||
|
||||
hidden_dim = w1.size(2) * 2
|
||||
intermediate_size = w1.size(1) // 2
|
||||
|
||||
return (
|
||||
w1,
|
||||
w1_scale,
|
||||
w1_gscale,
|
||||
w2,
|
||||
w2_scale,
|
||||
w2_gscale,
|
||||
a1_gscale,
|
||||
a2_gscale,
|
||||
num_experts,
|
||||
hidden_dim,
|
||||
intermediate_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Triton NVFP4 kernel requires CUDA.",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 1024])
|
||||
@pytest.mark.parametrize("top_k", [4])
|
||||
@pytest.mark.parametrize("model_id", list(MOE_MODEL_CONFIGS.keys()))
|
||||
@pytest.mark.parametrize(
|
||||
"tensor_parallel_size",
|
||||
[pytest.param(val, id=f"tensor_parallel_size:{val}") for val in [1, 2, 4, 8]],
|
||||
)
|
||||
def test_nvfp4_moe_correctness(
|
||||
loaded_model_files,
|
||||
num_tokens: int,
|
||||
top_k: int,
|
||||
model_id: str,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
"""Compare Nvfp4QuantizationEmulationTritonExperts (fused weight dequant + compute)
|
||||
against the unfused reference Nvfp4QuantizationEmulationTritonExpertsReference.
|
||||
|
||||
Both must produce bit-identical results.
|
||||
"""
|
||||
num_test_experts = max(8, top_k)
|
||||
(
|
||||
w1,
|
||||
w1_scale,
|
||||
w1_gscale,
|
||||
w2,
|
||||
w2_scale,
|
||||
w2_gscale,
|
||||
a1_gscale,
|
||||
a2_gscale,
|
||||
num_experts,
|
||||
hidden_dim,
|
||||
intermediate_size,
|
||||
) = _load_nvfp4_moe_weights(
|
||||
loaded_model_files,
|
||||
model_id,
|
||||
tensor_parallel_size,
|
||||
max_experts=num_test_experts,
|
||||
)
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=top_k,
|
||||
hidden_dim=hidden_dim,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=512,
|
||||
)
|
||||
|
||||
def _make_quant_config():
|
||||
return nvfp4_moe_quant_config(
|
||||
g1_alphas=w1_gscale.clone(),
|
||||
g2_alphas=w2_gscale.clone(),
|
||||
a1_gscale=a1_gscale.clone(),
|
||||
a2_gscale=a2_gscale.clone(),
|
||||
w1_scale=w1_scale.clone(),
|
||||
w2_scale=w2_scale.clone(),
|
||||
)
|
||||
|
||||
ref_experts = Nvfp4QuantizationEmulationTritonExpertsReference(
|
||||
moe_config=moe_config,
|
||||
quant_config=_make_quant_config(),
|
||||
)
|
||||
fused_experts = Nvfp4QuantizationEmulationTritonExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=_make_quant_config(),
|
||||
)
|
||||
|
||||
torch.manual_seed(42)
|
||||
hidden_states = torch.randn(
|
||||
num_tokens, hidden_dim, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights = torch.randn(
|
||||
num_tokens, top_k, dtype=torch.float32, device="cuda"
|
||||
).softmax(dim=-1)
|
||||
topk_ids = torch.stack(
|
||||
[torch.randperm(num_experts, device="cuda")[:top_k] for _ in range(num_tokens)]
|
||||
).to(torch.int32)
|
||||
|
||||
N = w1.size(1) # 2 * intermediate
|
||||
K = hidden_dim
|
||||
|
||||
ws13_size = num_tokens * top_k * max(intermediate_size, K)
|
||||
ws2_size = num_tokens * top_k * max(N, K)
|
||||
|
||||
workspace13_ref = torch.zeros(ws13_size, dtype=torch.bfloat16, device="cuda")
|
||||
workspace2_ref = torch.zeros(ws2_size, dtype=torch.bfloat16, device="cuda")
|
||||
output_ref = torch.zeros(num_tokens, K, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
workspace13_fused = torch.zeros_like(workspace13_ref)
|
||||
workspace2_fused = torch.zeros_like(workspace2_ref)
|
||||
output_fused = torch.zeros_like(output_ref)
|
||||
|
||||
apply_kwargs = dict(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# Unfused reference.
|
||||
ref_experts.apply(
|
||||
output=output_ref,
|
||||
workspace13=workspace13_ref,
|
||||
workspace2=workspace2_ref,
|
||||
**apply_kwargs,
|
||||
)
|
||||
|
||||
# Fused implementation.
|
||||
fused_experts.apply(
|
||||
output=output_fused,
|
||||
workspace13=workspace13_fused,
|
||||
workspace2=workspace2_fused,
|
||||
**apply_kwargs,
|
||||
)
|
||||
|
||||
# Not strict equality on H100, MI325, MI300 (< 0.1% elements).
|
||||
# The fused on-the-fly dequant path can lower to a slightly
|
||||
# different Triton/MMA tiling than the pre-dequantized
|
||||
# reference; experiments with reference-like tiling/masking
|
||||
# reduced some diffs were not kept because they regress
|
||||
# the fused kernel speed.
|
||||
# Strict equality validated on MI355.
|
||||
torch.testing.assert_close(
|
||||
output_fused,
|
||||
output_ref,
|
||||
atol=0.0 if on_gfx950() else 0.02,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
def _dense_nvfp4_reference(
|
||||
layer: torch.nn.Module, x: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""Decode E2M1 nibbles and both scale levels without production helpers."""
|
||||
levels = kE2M1ToFloat.cpu()
|
||||
weight = break_fp4_bytes(layer.weight.cpu(), torch.float32)
|
||||
weight *= (
|
||||
layer.weight_scale.cpu().float() * layer.weight_global_scale.cpu()
|
||||
).repeat_interleave(16, dim=-1)
|
||||
blocks = x.cpu().float().reshape(x.shape[0], -1, 16)
|
||||
global_scale = layer.input_global_scale_inv.cpu()
|
||||
scales = (
|
||||
(blocks.abs().amax(-1, keepdim=True) * (1.0 / FLOAT4_E2M1_MAX) * global_scale)
|
||||
.clamp(max=FLOAT8_E4M3_MAX)
|
||||
.to(torch.float8_e4m3fn)
|
||||
.float()
|
||||
)
|
||||
inverse = torch.where(scales == 0, 0, global_scale / scales)
|
||||
scaled = (blocks * inverse).abs().clamp(max=FLOAT4_E2M1_MAX)
|
||||
distances = (scaled.unsqueeze(-1) - levels).abs()
|
||||
# E2M1 ties round to the even significand (even encoding).
|
||||
nearest_mask = distances == distances.amin(-1, keepdim=True)
|
||||
even_mask = nearest_mask & (torch.arange(8, device="cpu") % 2 == 0)
|
||||
candidates = torch.where(even_mask.any(-1, keepdim=True), even_mask, nearest_mask)
|
||||
nearest = candidates.to(torch.int8).argmax(-1)
|
||||
activation = levels[nearest] * blocks.sign() * (scales / global_scale)
|
||||
expected = activation.reshape_as(x).to(x.dtype) @ weight.to(x.dtype).T
|
||||
if bias is not None:
|
||||
expected += bias.cpu()
|
||||
return expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm NVFP4 fallback")
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("m,n,k", [(1, 32, 64), (17, 128, 256)])
|
||||
@pytest.mark.parametrize("with_bias", [False, True])
|
||||
def test_nvfp4_dense_emulation(
|
||||
dtype: torch.dtype, m: int, n: int, k: int, with_bias: bool
|
||||
) -> None:
|
||||
"""W4A4 fallback preserves packed scales, bias and outputs across replay."""
|
||||
set_random_seed(47)
|
||||
layer = torch.nn.Module()
|
||||
# Every signed E2M1 code appears, including both zero encodings.
|
||||
codes = (torch.randperm(n * k, device="cuda") % 16).reshape(n, k)
|
||||
layer.weight = (codes[:, ::2] | (codes[:, 1::2] << 4)).to(torch.uint8)
|
||||
layer.weight_scale = (
|
||||
torch.arange(n * k // 16, device="cuda").reshape(n, k // 16) % 7 / 8
|
||||
).to(torch.float8_e4m3fn)
|
||||
layer.weight_global_scale = torch.tensor(0.25, device="cuda")
|
||||
layer.input_global_scale_inv = torch.tensor(3.0, device="cuda")
|
||||
x = torch.randn(m, k, device="cuda", dtype=dtype)
|
||||
x[:, :16] = 0
|
||||
bias = torch.randn(n, device="cuda", dtype=dtype) if with_bias else None
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(kernel_config=KernelConfig(linear_backend="emulation"))
|
||||
):
|
||||
kernel = init_nvfp4_linear_kernel()
|
||||
assert isinstance(kernel, EmulationNvFp4LinearKernel)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
original_weight = layer.weight.clone()
|
||||
original_scale = layer.weight_scale.view(torch.uint8).clone()
|
||||
actual = kernel.apply_weights(layer, x, bias)
|
||||
torch.testing.assert_close(
|
||||
actual.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
|
||||
)
|
||||
assert actual.abs().max() > 0.1
|
||||
saved = actual.clone()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = kernel.apply_weights(layer, x, bias)
|
||||
for sign in (-1, 1):
|
||||
x.mul_(sign * 0.75)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(
|
||||
captured.cpu(), _dense_nvfp4_reference(layer, x, bias), atol=0.03, rtol=0.01
|
||||
)
|
||||
torch.testing.assert_close(actual, saved, atol=0, rtol=0)
|
||||
assert torch.equal(layer.weight, original_weight)
|
||||
assert torch.equal(layer.weight_scale.view(torch.uint8), original_scale)
|
||||
@@ -1,32 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for NVFP4 linear kernel selection order (CPU-only)."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
_POSSIBLE_NVFP4_KERNELS,
|
||||
CutlassNvFp4LinearKernel,
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.platforms.interface import PlatformEnum
|
||||
|
||||
# W4A4 kernels that run on SM120/121, where the head of the list is gated to
|
||||
# sm_10x and selection falls through to whatever follows.
|
||||
W4A4_KERNELS_ON_SM12X = (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
FlashInferB12xNvFp4LinearKernel,
|
||||
CutlassNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("w4a4_kernel", W4A4_KERNELS_ON_SM12X)
|
||||
def test_w4a16_kernel_does_not_precede_w4a4_kernels(w4a4_kernel):
|
||||
candidates = _POSSIBLE_NVFP4_KERNELS[PlatformEnum.CUDA]
|
||||
w4a16_index = candidates.index(FlashInferCuteDslNvFp4W4A16LinearKernel)
|
||||
assert candidates.index(w4a4_kernel) < w4a16_index, (
|
||||
f"{w4a4_kernel.__name__} must be preferred over "
|
||||
f"{FlashInferCuteDslNvFp4W4A16LinearKernel.__name__}"
|
||||
)
|
||||
@@ -1,346 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
reason="Nvfp4 Requires compute capability of 10 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
SHAPES = [(128, 64), (128, 128), (256, 64), (256, 128)]
|
||||
PAD_SHAPES = [
|
||||
(90, 64),
|
||||
(150, 64),
|
||||
(128, 48),
|
||||
(128, 80),
|
||||
(150, 80),
|
||||
(90, 48),
|
||||
(90, 128),
|
||||
(150, 128),
|
||||
(150, 48),
|
||||
(90, 80),
|
||||
(128, 512),
|
||||
(128, 1024),
|
||||
(128, 2048),
|
||||
(64, 7168),
|
||||
(64, 7152),
|
||||
(32, 14336),
|
||||
]
|
||||
PADDED_OUTPUT_SHAPES = [(128, 48), (128, 80), (150, 48), (150, 80), (64, 7152)]
|
||||
SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
# E2M1 to float
|
||||
# 0111 -> 6
|
||||
# 0110 -> 4
|
||||
# 0101 -> 3
|
||||
# 0100 -> 2
|
||||
# 0011 -> 1.5
|
||||
# 0010 -> 1
|
||||
# 0001 -> 0.5
|
||||
# 0000 -> 0
|
||||
E2M1_TO_FLOAT32 = [
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
]
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
|
||||
def cast_from_fp4(x, m, n):
|
||||
# The fp4 values are packed in uint8 as [v_1st | v_2nd]
|
||||
v_2nd = x & 0xF
|
||||
v_1st = (x >> 4) & 0xF
|
||||
c = torch.stack((v_2nd, v_1st), dim=-1)
|
||||
out = torch.tensor([E2M1_TO_FLOAT32[x] for x in c.flatten()])
|
||||
out = out.reshape(m, n).to(torch.float32)
|
||||
return out
|
||||
|
||||
|
||||
def cast_to_fp4(x):
|
||||
sign = torch.sign(x)
|
||||
x = torch.abs(x)
|
||||
x[(x >= 0.0) & (x <= 0.25)] = 0.0
|
||||
x[(x > 0.25) & (x < 0.75)] = 0.5
|
||||
x[(x >= 0.75) & (x <= 1.25)] = 1.0
|
||||
x[(x > 1.25) & (x < 1.75)] = 1.5
|
||||
x[(x >= 1.75) & (x <= 2.5)] = 2.0
|
||||
x[(x > 2.5) & (x < 3.5)] = 3.0
|
||||
x[(x >= 3.5) & (x <= 5.0)] = 4.0
|
||||
x[x > 5.0] = 6.0
|
||||
return x * sign
|
||||
|
||||
|
||||
def get_reciprocal(x):
|
||||
if isinstance(x, torch.Tensor):
|
||||
return torch.where(x == 0, torch.tensor(0.0, dtype=x.dtype), 1.0 / x)
|
||||
elif isinstance(x, (float, int)):
|
||||
return 0.0 if x == 0 else 1.0 / x
|
||||
else:
|
||||
raise TypeError("Input must be a float, int, or a torch.Tensor.")
|
||||
|
||||
|
||||
def ref_nvfp4_quant(x, global_scale):
|
||||
assert global_scale.dtype == torch.float32
|
||||
assert x.ndim == 2
|
||||
m, n = x.shape
|
||||
x = torch.reshape(x, (m, n // BLOCK_SIZE, BLOCK_SIZE))
|
||||
vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
scale = global_scale * (vec_max * get_reciprocal(FLOAT4_E2M1_MAX))
|
||||
scale = scale.to(torch.float8_e4m3fn).to(torch.float32)
|
||||
output_scale = get_reciprocal(scale * get_reciprocal(global_scale))
|
||||
|
||||
scaled_x = x.to(torch.float32) * output_scale
|
||||
clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
|
||||
return cast_to_fp4(clipped_x), scale.squeeze(-1)
|
||||
|
||||
|
||||
def recover_swizzled_scales(scale, m, n, include_padding=False):
|
||||
round_up = lambda x, y: (x + y - 1) // y * y
|
||||
rounded_m = round_up(m, 128)
|
||||
scale_n = n // BLOCK_SIZE
|
||||
rounded_n = round_up(scale_n, 4)
|
||||
# Recover the swizzled scaling factor to linear layout
|
||||
tmp = torch.reshape(scale, (1, rounded_m // 128, rounded_n // 4, 32, 4, 4))
|
||||
tmp = torch.permute(tmp, (0, 1, 4, 3, 2, 5))
|
||||
result = torch.reshape(tmp, (rounded_m, rounded_n)).to(torch.float32)
|
||||
if include_padding:
|
||||
return result
|
||||
return result[:m, :scale_n]
|
||||
|
||||
|
||||
def round_up(x: int, y: int) -> int:
|
||||
return (x + y - 1) // y * y
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4(
|
||||
dtype: torch.dtype,
|
||||
shape: tuple[int, int],
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
torch.set_default_device(device)
|
||||
|
||||
m, n = shape
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype)
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = ops.scaled_fp4_quant(x, global_scale)
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, n)
|
||||
out_ans = cast_from_fp4(out, m, n)
|
||||
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
torch.testing.assert_close(scale_ans, scale_ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[(32, 4096), (128, 4096), (1, 64), (127, 1024), (256, 16384)],
|
||||
)
|
||||
@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_python_util_matches_cpp_allocation(
|
||||
shape: tuple[int, int],
|
||||
is_sf_swizzled_layout: bool,
|
||||
) -> None:
|
||||
"""Verify that the Python utility (create_fp4_output_tensors) allocates
|
||||
tensors with the same shapes and dtypes as the C++ functional variant
|
||||
(scaled_fp4_quant_func).
|
||||
"""
|
||||
from vllm._custom_ops import create_fp4_output_tensors
|
||||
|
||||
torch.set_default_device("cuda:0")
|
||||
m, n = shape
|
||||
input_tensor = torch.randn((m, n), dtype=torch.bfloat16)
|
||||
input_scale = torch.tensor([1.0], dtype=torch.float32, device="cuda:0")
|
||||
|
||||
# C++ functional variant allocates internally
|
||||
cpp_out, cpp_scale = torch.ops._C.scaled_fp4_quant(
|
||||
input_tensor, input_scale, is_sf_swizzled_layout
|
||||
)
|
||||
|
||||
# Python utility
|
||||
py_out, py_scale = create_fp4_output_tensors(
|
||||
m, n, torch.device("cuda:0"), is_sf_swizzled_layout
|
||||
)
|
||||
|
||||
assert py_out.shape == cpp_out.shape, (
|
||||
f"Output shape mismatch: Python {py_out.shape} vs C++ {cpp_out.shape}"
|
||||
)
|
||||
assert py_out.dtype == cpp_out.dtype, (
|
||||
f"Output dtype mismatch: Python {py_out.dtype} vs C++ {cpp_out.dtype}"
|
||||
)
|
||||
assert py_scale.shape == cpp_scale.shape, (
|
||||
f"Scale shape mismatch: Python {py_scale.shape} vs C++ {cpp_scale.shape}"
|
||||
)
|
||||
assert py_scale.dtype == cpp_scale.dtype, (
|
||||
f"Scale dtype mismatch: Python {py_scale.dtype} vs C++ {cpp_scale.dtype}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", PADDED_OUTPUT_SHAPES)
|
||||
@pytest.mark.parametrize("is_sf_swizzled_layout", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_with_padded_output(
|
||||
shape: tuple[int, int],
|
||||
is_sf_swizzled_layout: bool,
|
||||
) -> None:
|
||||
from vllm._custom_ops import create_fp4_output_tensors
|
||||
|
||||
dtype = torch.float16
|
||||
set_random_seed(42)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
m, n = shape
|
||||
padded_n = round_up(n, 32)
|
||||
assert padded_n > n
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype)
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = ops.scaled_fp4_quant(
|
||||
x,
|
||||
global_scale,
|
||||
is_sf_swizzled_layout=is_sf_swizzled_layout,
|
||||
padded_n=padded_n,
|
||||
)
|
||||
py_out, py_scale = create_fp4_output_tensors(
|
||||
m,
|
||||
n,
|
||||
torch.device("cuda:0"),
|
||||
is_sf_swizzled_layout,
|
||||
padded_n=padded_n,
|
||||
)
|
||||
|
||||
assert out.shape == (m, padded_n // 2)
|
||||
assert out.shape == py_out.shape
|
||||
assert out_scale.shape == py_scale.view(torch.float8_e4m3fn).shape
|
||||
|
||||
out_ans = cast_from_fp4(out[:, : n // 2], m, n)
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
assert torch.count_nonzero(out[:, n // 2 :]) == 0
|
||||
|
||||
if is_sf_swizzled_layout:
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, padded_n)
|
||||
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
|
||||
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
|
||||
else:
|
||||
scale_ans = out_scale.to(torch.float32)
|
||||
torch.testing.assert_close(scale_ans[:, : n // BLOCK_SIZE], scale_ref)
|
||||
assert torch.count_nonzero(scale_ans[:, n // BLOCK_SIZE :]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "padded_n"),
|
||||
[((90, 80), None), ((90, 48), 64)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_swizzled_scale_padding_initialized_in_kernel(
|
||||
shape: tuple[int, int], padded_n: int | None
|
||||
) -> None:
|
||||
"""The out variant must overwrite dirty scale padding without a memset."""
|
||||
torch.set_default_device("cuda:0")
|
||||
m, n = shape
|
||||
physical_n = padded_n if padded_n is not None else n
|
||||
rounded_m = round_up(m, 128)
|
||||
rounded_scale_n = round_up(physical_n // BLOCK_SIZE, 4)
|
||||
|
||||
x = torch.randn((m, n), dtype=torch.float16)
|
||||
global_scale = torch.tensor(1.0, dtype=torch.float32)
|
||||
out = torch.full((m, physical_n // 2), 0xFF, dtype=torch.uint8)
|
||||
out_scale = torch.full(
|
||||
(rounded_m, rounded_scale_n // 4), 0x7F7F7F7F, dtype=torch.int32
|
||||
)
|
||||
|
||||
torch.ops._C.scaled_fp4_quant.out(
|
||||
x,
|
||||
global_scale,
|
||||
True,
|
||||
output=out,
|
||||
output_scale=out_scale,
|
||||
)
|
||||
|
||||
full_scale = recover_swizzled_scales(
|
||||
out_scale.view(torch.float8_e4m3fn), m, physical_n, include_padding=True
|
||||
)
|
||||
assert torch.count_nonzero(full_scale[m:, :]) == 0
|
||||
assert torch.count_nonzero(full_scale[:m, n // BLOCK_SIZE :]) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_padded(pad_shape: tuple[int, int]) -> None:
|
||||
dtype = torch.float16
|
||||
set_random_seed(42)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
m, n = pad_shape
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype)
|
||||
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = ops.scaled_fp4_quant(x, global_scale)
|
||||
scale_ans = recover_swizzled_scales(out_scale, m, n)
|
||||
out_ans = cast_from_fp4(out, m, n)
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
torch.testing.assert_close(scale_ans, scale_ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pad_shape", PAD_SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_quantize_to_fp4_padded_no_sf_swizzled(pad_shape: tuple[int, int]) -> None:
|
||||
dtype = torch.float16
|
||||
set_random_seed(42)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
m, n = pad_shape
|
||||
|
||||
x = torch.randn((m, n), dtype=dtype)
|
||||
|
||||
tensor_amax = torch.abs(x).max().to(torch.float32)
|
||||
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / tensor_amax
|
||||
out_ref, scale_ref = ref_nvfp4_quant(x, global_scale)
|
||||
|
||||
out, out_scale = ops.scaled_fp4_quant(x, global_scale, is_sf_swizzled_layout=False)
|
||||
scale_ans = out_scale.to(torch.float32)
|
||||
out_ans = cast_from_fp4(out, m, n)
|
||||
torch.testing.assert_close(out_ans, out_ref)
|
||||
torch.testing.assert_close(scale_ans, scale_ref)
|
||||
@@ -1,269 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Copyright (C) 2025 Roberto L. Castro (Roberto.LopezCastro@ist.ac.at).
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.transform.utils.hadamard import deterministic_hadamard_matrix
|
||||
|
||||
from vllm import _custom_ops as ops # use existing nvfp4 gemm in vllm
|
||||
from vllm._custom_ops import fusedQuantizeNv
|
||||
from vllm.model_executor.layers.quantization.qutlass_utils import to_blocked
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required for these tests.", allow_module_level=True)
|
||||
|
||||
if not (
|
||||
current_platform.has_device_capability(100)
|
||||
or current_platform.has_device_capability(120)
|
||||
):
|
||||
pytest.skip(
|
||||
reason="Tests require compute capability 10.0 (100) or 12.0 (120).",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
# ----- Helpers -----
|
||||
def get_hadamard_matrix(group_size: int, dtype: torch.dtype, device: torch.device):
|
||||
return (
|
||||
deterministic_hadamard_matrix(group_size, dtype=dtype, device=device)
|
||||
* group_size**-0.5
|
||||
)
|
||||
|
||||
|
||||
def _rtne_fp4(x: torch.Tensor):
|
||||
device = x.device
|
||||
grid = torch.tensor(
|
||||
[
|
||||
-6.0,
|
||||
-4.0,
|
||||
-3.0,
|
||||
-2.0,
|
||||
-1.5,
|
||||
-1.0,
|
||||
-0.5,
|
||||
-0.0,
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
],
|
||||
dtype=x.dtype,
|
||||
device=x.device,
|
||||
)
|
||||
grid_int = torch.tensor(
|
||||
[-1, -2, -3, -4, -5, -6, -7, -8, 0, 1, 2, 3, 4, 5, 6, 7],
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
inds = torch.bucketize(x, grid)
|
||||
lo, hi = (inds - 1).clamp(min=0, max=15), inds.clamp(min=0, max=15)
|
||||
g_lo, g_hi = grid[lo], grid[hi]
|
||||
pick_hi = (g_hi - x < x - g_lo) | (g_hi - x == x - g_lo) & (grid_int[hi] % 2 == 0)
|
||||
y = torch.where(pick_hi, g_hi, g_lo)
|
||||
y_int = torch.where(pick_hi, grid_int[hi], grid_int[lo])
|
||||
y_int_packed = (y_int[..., 1::2] & 0xF) << 4 | y_int[..., ::2] & 0xF
|
||||
return y, y_int_packed
|
||||
|
||||
|
||||
def _dq_fp4(x_e2m1: torch.Tensor, x_e4m3: torch.Tensor, alpha: float):
|
||||
device = x_e2m1.device
|
||||
|
||||
x_e2m1_i32 = x_e2m1.view(dtype=torch.uint8).to(dtype=torch.int32)
|
||||
x_e2m1_unpacked = torch.stack(
|
||||
[x_e2m1_i32 & 0xF, (x_e2m1_i32 >> 4) & 0xF], dim=-1
|
||||
).flatten(start_dim=-2)
|
||||
|
||||
grid_dq = torch.tensor(
|
||||
[
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
],
|
||||
dtype=torch.float64,
|
||||
device=device,
|
||||
)
|
||||
x_fp4_dq = grid_dq[x_e2m1_unpacked]
|
||||
|
||||
scales_dq = x_e4m3.to(torch.float64)
|
||||
x_dq = (x_fp4_dq.unflatten(dim=-1, sizes=(-1, 16)) * scales_dq[..., None]).flatten(
|
||||
start_dim=-2
|
||||
) / alpha # * (4. / 3.)
|
||||
return x_dq, x_fp4_dq, scales_dq
|
||||
|
||||
|
||||
def _unpack_mask(clip_mask: torch.Tensor) -> torch.Tensor:
|
||||
clip_mask_unpacked_dq = torch.zeros(
|
||||
*clip_mask.shape[:-1],
|
||||
clip_mask.size(-1) * 8,
|
||||
dtype=torch.bool,
|
||||
device=clip_mask.device,
|
||||
)
|
||||
for i in range(8):
|
||||
clip_mask_unpacked_dq[..., i::8] = (clip_mask >> i) & 1
|
||||
return clip_mask_unpacked_dq
|
||||
|
||||
|
||||
def _forward_quantize_ref(x: torch.Tensor, h: torch.Tensor, rot_size: int):
|
||||
device = x.device
|
||||
|
||||
xh_ref64 = (
|
||||
x.unflatten(dim=-1, sizes=(-1, rot_size)).to(dtype=torch.float64)
|
||||
@ h.reshape(rot_size, rot_size).to(dtype=torch.float64)
|
||||
).flatten(start_dim=-2)
|
||||
|
||||
abs_max = xh_ref64.unflatten(dim=-1, sizes=(-1, 16)).abs().amax(dim=-1)
|
||||
scales_ref64_ = abs_max + 1e-8
|
||||
|
||||
xh_e4m3_ref = scales_ref64_.to(dtype=torch.float8_e4m3fn)
|
||||
scales_ref64 = xh_e4m3_ref.to(dtype=torch.float64)
|
||||
xh_scaled_ref64 = (
|
||||
xh_ref64.unflatten(dim=-1, sizes=(-1, 16)) / scales_ref64[..., None]
|
||||
).flatten(start_dim=-2)
|
||||
|
||||
xh_scaled_ref64 *= 6.0
|
||||
|
||||
clip_mask_unpacked_ref = xh_scaled_ref64.abs() < 6.0
|
||||
clip_mask_ref = torch.zeros(
|
||||
*x.shape[:-1], x.size(-1) // 8, dtype=torch.uint8, device=device
|
||||
)
|
||||
for i in range(8):
|
||||
clip_mask_ref |= clip_mask_unpacked_ref[..., i::8].to(dtype=torch.uint8) << i
|
||||
|
||||
xh_fp4_ref, xh_e2m1_ref = _rtne_fp4(xh_scaled_ref64)
|
||||
xh_dq, xh_fp4_dq, scales_dq = _dq_fp4(xh_e2m1_ref, xh_e4m3_ref, 6.0)
|
||||
clip_mask_unpacked_dq = _unpack_mask(clip_mask_ref)
|
||||
|
||||
assert xh_fp4_dq.equal(xh_fp4_ref)
|
||||
assert scales_dq.equal(scales_ref64)
|
||||
assert clip_mask_unpacked_dq.equal(clip_mask_unpacked_ref)
|
||||
|
||||
return (
|
||||
xh_dq,
|
||||
clip_mask_unpacked_ref,
|
||||
(xh_e2m1_ref, xh_e4m3_ref, clip_mask_ref),
|
||||
)
|
||||
|
||||
|
||||
DTYPE = torch.bfloat16
|
||||
DEVICE = torch.device("cuda:0")
|
||||
ROT_SIZES = [16, 32, 64, 128]
|
||||
GLOBAL_SCALES = [6.0]
|
||||
|
||||
LLAMA_MODELS = {
|
||||
"7B": [(4096, 3 * 4096), (4096, 4096), (4096, 2 * 10752), (10752, 4096)],
|
||||
"13B": [(5120, 3 * 5120), (5120, 5120), (5120, 2 * 13568), (13568, 5120)],
|
||||
"33B": [(6656, 3 * 6656), (6656, 6656), (6656, 2 * 17664), (17664, 6656)],
|
||||
"70B": [(8192, 3 * 8192), (8192, 8192), (8192, 2 * 21760), (21760, 8192)],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _seed_each_test():
|
||||
set_random_seed(0)
|
||||
np.random.seed(0)
|
||||
torch.random.manual_seed(0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rot_size", ROT_SIZES)
|
||||
@pytest.mark.parametrize("global_scale_value", GLOBAL_SCALES)
|
||||
@torch.inference_mode()
|
||||
def test_fused_quantization(rot_size: int, global_scale_value: float):
|
||||
dtype, device = DTYPE, DEVICE
|
||||
h = get_hadamard_matrix(rot_size, dtype, device)
|
||||
x = torch.randn(2, 4096, 4096, dtype=dtype, device=device) * 25.0
|
||||
global_scale = torch.tensor([global_scale_value], device=device)
|
||||
|
||||
xh_dq_ref, _, _ = _forward_quantize_ref(x, h, rot_size)
|
||||
xh_e2m1, xh_e4m3 = fusedQuantizeNv(x, h, global_scale)
|
||||
xh_e4m3 = xh_e4m3.reshape(2, 4096, 4096 // 16)
|
||||
xh_dq, *_ = _dq_fp4(xh_e2m1, xh_e4m3, alpha=global_scale_value)
|
||||
|
||||
torch.testing.assert_close(xh_dq, xh_dq_ref, rtol=0.34, atol=100)
|
||||
assert (xh_dq != xh_dq_ref).float().mean() <= 1e-1
|
||||
|
||||
m, n, k = 504, 4096 * 2, 4096
|
||||
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
|
||||
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
|
||||
|
||||
a_e2m1, a_e4m3 = fusedQuantizeNv(a, h, global_scale)
|
||||
b_e2m1, b_e4m3 = fusedQuantizeNv(b, h, global_scale)
|
||||
|
||||
a_dq, *_ = _dq_fp4(a_e2m1, a_e4m3[:m, :k], alpha=1.0)
|
||||
b_dq, *_ = _dq_fp4(b_e2m1, b_e4m3[:n, :k], alpha=1.0)
|
||||
out_ref = a_dq @ b_dq.transpose(-2, -1)
|
||||
|
||||
a_scale_block = to_blocked(a_e4m3, backend="triton").view(-1, k // 16)
|
||||
b_scale_block = to_blocked(b_e4m3, backend="triton").view(-1, k // 16)
|
||||
alpha = torch.tensor([1.0], device=device)
|
||||
out = ops.cutlass_scaled_fp4_mm(
|
||||
a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha, torch.bfloat16
|
||||
)
|
||||
assert out.equal(out_ref.to(dtype=out.dtype))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", list(LLAMA_MODELS.keys()))
|
||||
@pytest.mark.parametrize("layer_idx", [0, 1, 2, 3])
|
||||
@pytest.mark.parametrize("batch", [1, 16])
|
||||
@pytest.mark.parametrize("rot_size", ROT_SIZES)
|
||||
@torch.inference_mode()
|
||||
def test_llama_shapes(model: str, layer_idx: int, batch: int, rot_size: int):
|
||||
dtype, device = DTYPE, DEVICE
|
||||
m = batch
|
||||
k, n = LLAMA_MODELS[model][layer_idx]
|
||||
|
||||
h = get_hadamard_matrix(rot_size, dtype, device)
|
||||
|
||||
a = torch.randn(m, k, dtype=dtype, device=device) * 25.0
|
||||
b = torch.randn(n, k, dtype=dtype, device=device) * 25.0
|
||||
|
||||
global_scale = torch.tensor([1.0], device=device)
|
||||
|
||||
a_e2m1, a_e4m3 = fusedQuantizeNv(a, h, global_scale)
|
||||
b_e2m1, b_e4m3 = fusedQuantizeNv(b, h, global_scale)
|
||||
|
||||
a_dq, *_ = _dq_fp4(a_e2m1, a_e4m3[:m, :k], alpha=1.0)
|
||||
b_dq, *_ = _dq_fp4(b_e2m1, b_e4m3[:n, :k], alpha=1.0)
|
||||
out_ref = a_dq @ b_dq.transpose(-2, -1)
|
||||
|
||||
a_scale_block = to_blocked(a_e4m3, backend="triton").view(-1, k // 16)
|
||||
b_scale_block = to_blocked(b_e4m3, backend="triton").view(-1, k // 16)
|
||||
alpha = torch.tensor([1.0], device=device)
|
||||
out = ops.cutlass_scaled_fp4_mm(
|
||||
a_e2m1, b_e2m1, a_scale_block, b_scale_block, alpha, torch.bfloat16
|
||||
)
|
||||
assert out.equal(out_ref.to(dtype=out.dtype))
|
||||
@@ -1,100 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, dequantize_nvfp4_to_dtype
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
reason="Nvfp4 Requires compute capability of 10 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
# m, n, k
|
||||
SHAPES = [(128, 128, 64), (128, 128, 128), (256, 128, 64), (128, 256, 128)]
|
||||
PAD_SHAPES = [(150, 128, 64), (128, 128, 96)]
|
||||
SHAPES.extend(PAD_SHAPES)
|
||||
|
||||
SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
def get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_sf,
|
||||
b_sf,
|
||||
a_global_scale,
|
||||
b_global_scale,
|
||||
m,
|
||||
n,
|
||||
dtype,
|
||||
block_size,
|
||||
device,
|
||||
):
|
||||
_, m_k = a_fp4.shape
|
||||
_, n_k = b_fp4.shape
|
||||
assert m_k == n_k
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4, a_sf, a_global_scale, dtype=dtype, device=device, block_size=block_size
|
||||
)
|
||||
b_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
b_fp4, b_sf, b_global_scale, dtype=dtype, device=device, block_size=block_size
|
||||
)
|
||||
return torch.matmul(a_in_dtype, b_in_dtype.t())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@torch.inference_mode()
|
||||
def test_nvfp4_gemm(
|
||||
dtype: torch.dtype,
|
||||
shape: tuple[int, int, int],
|
||||
seed: int,
|
||||
device: str,
|
||||
) -> None:
|
||||
set_random_seed(seed)
|
||||
m, n, packed_k = shape
|
||||
k = packed_k * 2
|
||||
block_size = 16
|
||||
a_dtype = torch.randn((m, k), dtype=dtype, device=device)
|
||||
b_dtype = torch.randn((n, k), dtype=dtype, device=device)
|
||||
|
||||
a_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(a_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
b_global_scale = (
|
||||
(FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.amax(b_dtype.flatten(), dim=-1)
|
||||
).to(torch.float32)
|
||||
alpha = 1.0 / (a_global_scale * b_global_scale)
|
||||
# ops.scaled_fp4_quant returns swizzled scales, while weights
|
||||
# from checkpoints are in linear scales.
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a_dtype, a_global_scale)
|
||||
b_fp4, b_scale_interleaved = ops.scaled_fp4_quant(b_dtype, b_global_scale)
|
||||
|
||||
# get_ref_results unswizzles the scales internally.
|
||||
expected_out = get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
a_scale_interleaved,
|
||||
b_scale_interleaved,
|
||||
a_global_scale,
|
||||
b_global_scale,
|
||||
m,
|
||||
n,
|
||||
dtype,
|
||||
block_size,
|
||||
device,
|
||||
)
|
||||
out = ops.cutlass_scaled_fp4_mm(
|
||||
a_fp4, b_fp4, a_scale_interleaved, b_scale_interleaved, alpha, dtype
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, expected_out.to(dtype=dtype), atol=1e-1, rtol=1e-1)
|
||||
@@ -1,541 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils import fp8_utils, int8_utils
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import get_fp8_min_max
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(0, 128),
|
||||
(1, 128),
|
||||
(3, 384),
|
||||
(5, 384),
|
||||
(7, 384),
|
||||
(15, 384),
|
||||
(31, 128),
|
||||
(32, 128),
|
||||
(63, 256),
|
||||
(64, 256),
|
||||
(16, 512),
|
||||
(3, 2688),
|
||||
(3, 5376),
|
||||
(333, 128),
|
||||
(512, 6144),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("column_major", [False, True])
|
||||
@pytest.mark.parametrize("tma_aligned", [False, True])
|
||||
@pytest.mark.parametrize("scale_ue8m0", [False, True])
|
||||
@pytest.mark.parametrize("group_size", [64, 128])
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Only test on CUDA/ROCm/XPU.",
|
||||
)
|
||||
def test_per_token_group_quant_fp8(
|
||||
shape,
|
||||
dtype: torch.dtype,
|
||||
column_major: bool,
|
||||
tma_aligned: bool,
|
||||
scale_ue8m0: bool,
|
||||
group_size: int,
|
||||
):
|
||||
"""Tail groups preserve FP8 values and scales in every supported layout."""
|
||||
device = current_platform.device_type
|
||||
|
||||
torch.manual_seed(42)
|
||||
num_tokens, hidden_dim = shape
|
||||
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=dtype) * 8
|
||||
|
||||
# native kernel path
|
||||
out_q, scale = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
column_major_scales=column_major,
|
||||
tma_aligned_scales=tma_aligned,
|
||||
use_ue8m0=scale_ue8m0,
|
||||
)
|
||||
|
||||
# triton ref
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False),
|
||||
patch("vllm.platforms.current_platform.is_xpu", return_value=False),
|
||||
):
|
||||
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
column_major_scales=column_major,
|
||||
use_ue8m0=scale_ue8m0,
|
||||
)
|
||||
|
||||
assert torch.allclose(out_q.float(), ref_q.float(), atol=0.15, rtol=0.15)
|
||||
assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01)
|
||||
|
||||
|
||||
def _misaligned_clone(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Contiguous copy of ``x`` whose ``data_ptr`` is not 16B aligned."""
|
||||
numel = x.numel()
|
||||
base = torch.empty(numel + 16 // x.element_size(), device=x.device, dtype=x.dtype)
|
||||
view = base[1 : 1 + numel]
|
||||
if view.data_ptr() % 16 == 0:
|
||||
pytest.skip("allocator did not yield a misaligned offset")
|
||||
return view.view(x.shape).copy_(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)])
|
||||
@pytest.mark.parametrize("column_major", [False, True])
|
||||
@pytest.mark.parametrize("scale_ue8m0", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="register-resident schedule is CUDA/ROCm only",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_register_matches_smem(
|
||||
shape, column_major: bool, scale_ue8m0: bool, dtype: torch.dtype
|
||||
):
|
||||
"""group_size=128 with 16-bit input takes a register-resident schedule that
|
||||
needs 16B-aligned buffers; a misaligned input falls back to the
|
||||
shared-memory schedule. The two must agree bit-for-bit, which the
|
||||
tolerance-based comparison against Triton above cannot establish.
|
||||
"""
|
||||
device = current_platform.device_type
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(shape, device=device, dtype=dtype) * 8
|
||||
|
||||
reg_q, reg_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x, 128, column_major_scales=column_major, use_ue8m0=scale_ue8m0
|
||||
)
|
||||
smem_q, smem_s = fp8_utils.per_token_group_quant_fp8(
|
||||
_misaligned_clone(x),
|
||||
128,
|
||||
column_major_scales=column_major,
|
||||
use_ue8m0=scale_ue8m0,
|
||||
)
|
||||
|
||||
assert torch.equal(reg_q.view(torch.uint8), smem_q.view(torch.uint8))
|
||||
assert torch.equal(reg_s, smem_s)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,hidden_dim,group_size",
|
||||
[
|
||||
# No padding: mn=4 (mult of 4), groups_per_row=56 (mult of 4)
|
||||
(4, 7168, 128),
|
||||
# MN padding only: mn=1, tma_aligned_mn=4
|
||||
(1, 7168, 128),
|
||||
# MN padding only: mn=3, tma_aligned_mn=4
|
||||
(3, 7168, 128),
|
||||
# K padding only: groups_per_row=5 (5%4=1)
|
||||
(4, 640, 128),
|
||||
# K padding only: groups_per_row=6 (6%4=2)
|
||||
(4, 768, 128),
|
||||
# Single packed column, no padding: k_num_packed=1, mn%4=0
|
||||
(4, 384, 128),
|
||||
# Both MN and K padding
|
||||
(1, 384, 128),
|
||||
(3, 640, 128),
|
||||
# Larger shapes with no padding
|
||||
(64, 7168, 128),
|
||||
(128, 14336, 128),
|
||||
# Larger shapes with padding
|
||||
(127, 7168, 128),
|
||||
(253, 640, 128),
|
||||
(1, 32, 32),
|
||||
(3, 64, 32),
|
||||
(7, 160, 32),
|
||||
(32, 512, 32),
|
||||
(127, 4096, 32),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("poisoned_scales", [False, True])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="DeepGEMM not available on this platform",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_packed(
|
||||
num_tokens, hidden_dim, group_size, poisoned_scales
|
||||
):
|
||||
"""Test the packed DeepGEMM quantization kernel against the Triton
|
||||
reference (row-major, UE8M0 scales)."""
|
||||
device = "cuda"
|
||||
torch.manual_seed(42)
|
||||
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
|
||||
|
||||
mn = num_tokens
|
||||
groups_per_row = hidden_dim // group_size
|
||||
k_num_packed = (groups_per_row + 3) // 4
|
||||
tma_aligned_mn = ((mn + 3) // 4) * 4
|
||||
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
|
||||
|
||||
if poisoned_scales:
|
||||
# Call the kernel with poisoned scale buffer to
|
||||
# ensure padded indices are correctly zeroed.
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
out_q = torch.empty_like(x, dtype=fp8_dtype)
|
||||
out_s_packed = torch.empty_strided(
|
||||
(mn, k_num_packed),
|
||||
(1, tma_aligned_mn),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).fill_(0x7F7F7F7F)
|
||||
torch.ops._C.per_token_group_fp8_quant_packed(
|
||||
x,
|
||||
out_q,
|
||||
out_s_packed,
|
||||
group_size,
|
||||
1e-10,
|
||||
fp8_min,
|
||||
fp8_max,
|
||||
)
|
||||
else:
|
||||
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x,
|
||||
group_size=group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
# Triton reference (row-major float32 scales, UE8M0)
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
# Quantized values must match.
|
||||
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
|
||||
|
||||
# Verify packed scales (valid exponents + padding zeros).
|
||||
ref_s_flat = ref_s.reshape(mn, groups_per_row)
|
||||
ref_exponents = (ref_s_flat.view(torch.int32) >> 23) & 0xFF
|
||||
|
||||
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
|
||||
for row in range(mn):
|
||||
for g in range(groups_per_row):
|
||||
pack_col = g // 4
|
||||
pos = g % 4
|
||||
idx = pack_col * tma_aligned_mn + row
|
||||
expected[idx] |= int(ref_exponents[row, g].item()) << (pos * 8)
|
||||
|
||||
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
|
||||
assert torch.equal(actual, expected), (
|
||||
f"Packed scale storage mismatch.\n"
|
||||
f"First diff at index "
|
||||
f"{(actual != expected).nonzero(as_tuple=True)[0][0].item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="DeepGEMM not available on this platform",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_packed_all_zero():
|
||||
"""All-zero input must produce well-defined UE8M0 scale bytes via the eps
|
||||
floor in the kernel's UE8M0 path. Locks down the all-zero behavior before
|
||||
optimization.
|
||||
|
||||
The CUDA kernel computes:
|
||||
y_s = eps / fp8_max
|
||||
y_s = exp2(ceil(log2(fmax(y_s, 1e-10))))
|
||||
For all-zero input, eps/fp8_max < 1e-10, so the inner fmax clamps back to
|
||||
1e-10, giving exp2(ceil(log2(1e-10))) = exp2(-33) => UE8M0 byte 0x5E (94).
|
||||
"""
|
||||
device = "cuda"
|
||||
num_tokens, hidden_dim, group_size = 4, 7168, 128
|
||||
x = torch.zeros((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16)
|
||||
|
||||
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x,
|
||||
group_size=group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
# Quantized values must be all zero.
|
||||
assert torch.equal(
|
||||
out_q.view(torch.uint8),
|
||||
torch.zeros_like(out_q, dtype=torch.uint8),
|
||||
), "All-zero input should produce all-zero FP8 output"
|
||||
|
||||
# UE8M0 byte produced by the kernel for all-zero input.
|
||||
# The kernel's inner fmax(y_s, 1e-10) clamps eps/fp8_max back to 1e-10.
|
||||
# 1e-10 as float32 has biased exponent 0x5D and a non-zero mantissa, so
|
||||
# the kernel's bit-twiddle (exp_bits + (mant_bits != 0)) rounds up to
|
||||
# 0x5E. This matches exp2(ceil(log2(1e-10))) = exp2(-33).
|
||||
expected_exp_byte = 0x5E
|
||||
|
||||
mn = num_tokens
|
||||
groups_per_row = hidden_dim // group_size
|
||||
k_num_packed = (groups_per_row + 3) // 4
|
||||
tma_aligned_mn = ((mn + 3) // 4) * 4
|
||||
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
|
||||
|
||||
# All valid scale slots must contain the expected packed value.
|
||||
# Padding slots must be zero.
|
||||
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
|
||||
|
||||
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
|
||||
for row in range(mn):
|
||||
for g in range(groups_per_row):
|
||||
pack_col = g // 4
|
||||
pos = g % 4
|
||||
idx = pack_col * tma_aligned_mn + row
|
||||
expected[idx] |= expected_exp_byte << (pos * 8)
|
||||
|
||||
assert torch.equal(actual, expected), "All-zero scale bytes mismatch"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="DeepGEMM not available on this platform",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_packed_mantissa_rounds_up():
|
||||
"""Inputs whose absmax/max_8bit produces a non-power-of-2 force the
|
||||
mantissa-rounding-up branch (exp_byte += 1). Locks down this behavior
|
||||
before optimization."""
|
||||
device = "cuda"
|
||||
num_tokens, hidden_dim, group_size = 4, 7168, 128
|
||||
|
||||
# Build a tensor whose per-group absmax = 1.5 * fp8_max * 2^k for various k.
|
||||
# fp8_max = torch.finfo(torch.float8_e4m3fn).max = 448.0.
|
||||
# Then absmax/fp8_max = 1.5 * 2^k -> non-zero mantissa, triggers ceil
|
||||
# rounding to 2^(k+1). Use k=0 for simplicity; the bf16 representation of
|
||||
# 1.5*448=672.0 is exact.
|
||||
x = torch.full(
|
||||
(num_tokens, hidden_dim),
|
||||
672.0,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x,
|
||||
group_size=group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x,
|
||||
group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
|
||||
|
||||
mn = num_tokens
|
||||
groups_per_row = hidden_dim // group_size
|
||||
k_num_packed = (groups_per_row + 3) // 4
|
||||
tma_aligned_mn = ((mn + 3) // 4) * 4
|
||||
num_scale_elems = mn + (k_num_packed - 1) * tma_aligned_mn
|
||||
|
||||
ref_s_flat = ref_s.reshape(mn, groups_per_row)
|
||||
ref_exponents = (ref_s_flat.view(torch.int32) >> 23) & 0xFF
|
||||
expected = torch.zeros(num_scale_elems, dtype=torch.int32, device="cpu")
|
||||
for row in range(mn):
|
||||
for g in range(groups_per_row):
|
||||
pack_col = g // 4
|
||||
pos = g % 4
|
||||
idx = pack_col * tma_aligned_mn + row
|
||||
expected[idx] |= int(ref_exponents[row, g].item()) << (pos * 8)
|
||||
|
||||
actual = torch.as_strided(out_s_packed, (num_scale_elems,), (1,)).cpu()
|
||||
assert torch.equal(actual, expected), "Scale bytes mismatch"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,hidden_dim",
|
||||
[
|
||||
(1, 7168), # mn padded 1 -> 4
|
||||
(2, 7168), # mn padded 2 -> 4
|
||||
(3, 7168), # mn padded 3 -> 4
|
||||
(5, 7168), # mn padded 5 -> 8
|
||||
(127, 7168), # mn padded 127 -> 128
|
||||
(253, 640), # both mn and groups padded
|
||||
(1, 384), # extreme: 1 group, 1 mn row -> both axes padded
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="DeepGEMM not available on this platform",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_packed_zero_fills_padded_output_q(
|
||||
num_tokens, hidden_dim
|
||||
):
|
||||
"""When output_q is allocated with shape (tma_aligned_mn, k) instead of
|
||||
(mn, k), the kernel must overwrite the padded mn rows with zeros so
|
||||
callers can use ``torch.empty`` instead of ``torch.zeros``."""
|
||||
device = "cuda"
|
||||
group_size = 128
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
|
||||
|
||||
mn = num_tokens
|
||||
groups_per_row = hidden_dim // group_size
|
||||
k_num_packed = (groups_per_row + 3) // 4
|
||||
tma_aligned_mn = ((mn + 3) // 4) * 4
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
fp8_min, fp8_max = get_fp8_min_max()
|
||||
# Allocate output_q with the padded mn extent and pre-fill with 0xFF
|
||||
# so the kernel cannot rely on a clean buffer.
|
||||
out_q = torch.empty((tma_aligned_mn, hidden_dim), device=device, dtype=fp8_dtype)
|
||||
out_q.view(torch.uint8).fill_(0xFF)
|
||||
|
||||
out_s_packed = torch.empty_strided(
|
||||
(mn, k_num_packed),
|
||||
(1, tma_aligned_mn),
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
torch.ops._C.per_token_group_fp8_quant_packed(
|
||||
x, out_q, out_s_packed, group_size, 1e-10, fp8_min, fp8_max
|
||||
)
|
||||
|
||||
# Live rows must match the Triton reference.
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
ref_q, _ = fp8_utils.per_token_group_quant_fp8(x, group_size, use_ue8m0=True)
|
||||
assert torch.equal(out_q[:mn], ref_q), "Live region mismatch"
|
||||
|
||||
# Padded rows must be all-zero; without this, downstream TMA loads would
|
||||
# see uninitialised data.
|
||||
if tma_aligned_mn > mn:
|
||||
padded_bytes = out_q[mn:tma_aligned_mn].view(torch.uint8)
|
||||
assert padded_bytes.eq(0).all(), (
|
||||
f"Padded rows [{mn}, {tma_aligned_mn}) not zeroed; "
|
||||
f"{padded_bytes.ne(0).sum().item()} non-zero bytes"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="packed FP8 per-token-group quant kernel requires a CUDA-alike GPU",
|
||||
)
|
||||
def test_per_token_group_quant_fp8_packed_large_mn():
|
||||
"""Regression test for https://github.com/vllm-project/vllm/issues/45099.
|
||||
|
||||
Some background: gridDim.x and gridDim.y have different limits of 2^31 - 1 and
|
||||
2^16 - 1, respectively.
|
||||
Prior code introduced a bug where it incorrectly assumed grid.x and y both have
|
||||
2^31 - 1 limits and mixed them up, which doesn't surface until the kernel is
|
||||
launched with a large mn that exceeds grid.y limit (2^16 - 1).
|
||||
|
||||
This issue doesn't surface often because each forward pass only processes a
|
||||
bounded token batch, not the full context.
|
||||
Quantizing tensors with more rows than that will fail at launch with
|
||||
"CUDA error: invalid argument".
|
||||
This is a differential test that compares fp8 output against Triton output
|
||||
reference when token size sits just above the gridDim.y 2^16 - 1 limit.
|
||||
"""
|
||||
device = "cuda"
|
||||
group_size = 128
|
||||
# hidden 2048 -> 2048/128 = 16 groups per row -> kx=16, ry=1: one grid row per mn
|
||||
# row, so any mn > 65535 overflowed grid.y before the fix.
|
||||
num_tokens, hidden_dim = 65537, 2048
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) * 8
|
||||
|
||||
out_q, out_s_packed = fp8_utils.per_token_group_quant_fp8_packed_for_deepgemm(
|
||||
x,
|
||||
group_size=group_size,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
ref_q, ref_s = fp8_utils.per_token_group_quant_fp8(
|
||||
x, group_size, use_ue8m0=True
|
||||
)
|
||||
|
||||
assert torch.equal(out_q, ref_q), "Quantized output mismatch"
|
||||
|
||||
# Vectorized packed-scale check; the per-element loop used by the smaller
|
||||
# tests is too slow at this size. groups_per_row is a multiple of 4 here,
|
||||
# so there is no K padding and the packed view lines up.
|
||||
mn = num_tokens
|
||||
groups_per_row = hidden_dim // group_size
|
||||
k_num_packed = (groups_per_row + 3) // 4
|
||||
assert groups_per_row % 4 == 0
|
||||
ref_exponents = (ref_s.reshape(mn, groups_per_row).view(torch.int32) >> 23) & 0xFF
|
||||
exp = ref_exponents.view(mn, k_num_packed, 4)
|
||||
expected = (
|
||||
exp[..., 0] | (exp[..., 1] << 8) | (exp[..., 2] << 16) | (exp[..., 3] << 24)
|
||||
)
|
||||
assert torch.equal(out_s_packed.cpu(), expected.cpu()), "Packed scale mismatch"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(0, 128),
|
||||
(1, 128),
|
||||
(3, 384),
|
||||
(5, 384),
|
||||
(7, 384),
|
||||
(15, 384),
|
||||
(32, 128),
|
||||
(64, 256),
|
||||
(16, 512),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("group_size", [64, 128])
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Only test on CUDA/ROCm/XPU.",
|
||||
)
|
||||
def test_per_token_group_quant_int8(shape, dtype, group_size: int):
|
||||
"""The shared launch policy also handles INT8 tail groups."""
|
||||
device = current_platform.device_type
|
||||
|
||||
torch.manual_seed(42)
|
||||
num_tokens, hidden_dim = shape
|
||||
|
||||
x = torch.randn((num_tokens, hidden_dim), device=device, dtype=dtype) * 8
|
||||
|
||||
# cuda path
|
||||
out_q, scale = int8_utils.per_token_group_quant_int8(
|
||||
x,
|
||||
group_size,
|
||||
)
|
||||
|
||||
# triton ref
|
||||
with patch("vllm.platforms.current_platform.is_cuda_alike", return_value=False):
|
||||
ref_q, ref_s = int8_utils.per_token_group_quant_int8(
|
||||
x,
|
||||
group_size,
|
||||
)
|
||||
|
||||
assert torch.allclose(out_q.float(), ref_q.float(), atol=0.15, rtol=0.15)
|
||||
assert torch.allclose(scale, ref_s, atol=0.01, rtol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 128), (333, 128), (1024, 6144)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="register-resident schedule is CUDA/ROCm only",
|
||||
)
|
||||
def test_per_token_group_quant_int8_register_matches_smem(shape, dtype: torch.dtype):
|
||||
"""int8 output shares the register-resident schedule with fp8, so it needs
|
||||
the same bit-exactness guard against the shared-memory schedule.
|
||||
"""
|
||||
device = current_platform.device_type
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(shape, device=device, dtype=dtype) * 8
|
||||
|
||||
reg_q, reg_s = int8_utils.per_token_group_quant_int8(x, 128)
|
||||
smem_q, smem_s = int8_utils.per_token_group_quant_int8(_misaligned_clone(x), 128)
|
||||
|
||||
assert torch.equal(reg_q, smem_q)
|
||||
assert torch.equal(reg_s, smem_s)
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Schema/aliasing tests for the AITER FP8 quantization custom ops.
|
||||
#
|
||||
# These use the shared opcheck helper, whose test_schema check catches custom
|
||||
# ops whose implementation aliases an input that the registered schema declares
|
||||
# as non-aliasing -- the failure mode behind the rocm_aiter_per_tensor_quant
|
||||
# regression (a returned scale that aliased the input scale).
|
||||
#
|
||||
# Skipped if AITER is not installed or the platform is not ROCm.
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import opcheck
|
||||
|
||||
# this import statement is needed to ensure the ops are registered
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and aiter_available),
|
||||
reason="AITER ops are only available on ROCm with aiter package installed",
|
||||
)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
def _x(M=128, N=4096):
|
||||
return torch.randn((M, N), dtype=torch.float16, device="cuda")
|
||||
|
||||
|
||||
def test_per_tensor_quant_static_schema():
|
||||
"""Static per-tensor: caller provides scale (the aliasing regression)."""
|
||||
x = _x()
|
||||
out = torch.empty_like(x, dtype=FP8_DTYPE)
|
||||
scale = torch.ones(1, dtype=torch.float32, device="cuda")
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant,
|
||||
(out, x, scale, False),
|
||||
)
|
||||
|
||||
|
||||
def test_per_tensor_quant_dynamic_schema():
|
||||
"""Dynamic per-tensor: op computes scale into the caller's buffer."""
|
||||
x = _x()
|
||||
out = torch.empty_like(x, dtype=FP8_DTYPE)
|
||||
scale = torch.empty(1, dtype=torch.float32, device="cuda")
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant,
|
||||
(out, x, scale, True),
|
||||
)
|
||||
|
||||
|
||||
def test_per_token_quant_dynamic_schema():
|
||||
"""Dynamic per-token: op computes scale into a freshly allocated buffer."""
|
||||
x = _x()
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_per_token_quant,
|
||||
(x, FP8_DTYPE, None),
|
||||
)
|
||||
|
||||
|
||||
def test_group_fp8_quant_schema():
|
||||
"""Dynamic per-token-group quant."""
|
||||
x = _x()
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_group_fp8_quant,
|
||||
(x, 128),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dynamic", [True, False])
|
||||
def test_per_tensor_quant_matches_native(dynamic):
|
||||
"""Wrapper output matches the native scaled_fp8_quant reference."""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
torch.manual_seed(0)
|
||||
x = _x()
|
||||
if dynamic:
|
||||
scale_in = None
|
||||
else:
|
||||
scale_in = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
|
||||
out, scale = rocm_aiter_ops.per_tensor_quant(x, FP8_DTYPE, scale_in)
|
||||
ref_out, ref_scale = ops.scaled_fp8_quant(x, scale_in)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
assert scale.shape == ref_scale.shape
|
||||
deq = out.to(torch.float32) * scale
|
||||
if dynamic:
|
||||
# Dynamic mode: AITER and native each compute their own scale, so their
|
||||
# outputs differ and can't be compared. Just check that AITER's output
|
||||
# dequantizes back to the input, within fp8 rounding error.
|
||||
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
|
||||
else:
|
||||
# Static mode: both use the caller's scale, so the outputs must match.
|
||||
assert torch.equal(scale, scale_in)
|
||||
ref_deq = ref_out.to(torch.float32) * ref_scale
|
||||
torch.testing.assert_close(deq, ref_deq, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_per_token_quant_matches_native():
|
||||
"""Per-token quant output dequantizes back to the input within FP8 error."""
|
||||
torch.manual_seed(0)
|
||||
x = _x()
|
||||
|
||||
out, scale = rocm_aiter_ops.per_token_quant(x, FP8_DTYPE)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
assert scale.shape[0] == x.shape[0]
|
||||
|
||||
# Dequantize and compare to original
|
||||
deq = out.to(torch.float32) * scale.view(-1, 1)
|
||||
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
|
||||
|
||||
|
||||
def test_rms_norm_determinism():
|
||||
"""AITER RMSNorm produces bitwise-identical results across repeated calls."""
|
||||
# Import to ensure ops are registered
|
||||
import vllm.kernels.aiter_ops # noqa: F401
|
||||
|
||||
torch.manual_seed(0)
|
||||
M, N = 32, 512
|
||||
x = torch.randn(M, N, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.ones(N, dtype=torch.bfloat16, device="cuda")
|
||||
eps = 1e-5
|
||||
|
||||
reference = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
|
||||
for i in range(3):
|
||||
result = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
|
||||
assert torch.equal(reference, result), f"Run {i + 1} differs from reference"
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Triton dequant-gather kernel used by
|
||||
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import unpack_from_int32
|
||||
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_embedding import ( # noqa: E501
|
||||
_dequant_gather_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _dequant_gather_torch(
|
||||
ids: torch.Tensor,
|
||||
weight_packed: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
hidden: int,
|
||||
num_bits: int,
|
||||
) -> torch.Tensor:
|
||||
"""Reference: gather packed rows by id, unpack int32-packed INT, dequant."""
|
||||
n = ids.shape[0]
|
||||
int8 = unpack_from_int32(weight_packed[ids], num_bits, torch.Size([n, hidden]))
|
||||
scale_rows = weight_scale[ids]
|
||||
w = int8.to(scale_rows.dtype)
|
||||
if scale_rows.shape[1] == 1:
|
||||
return w * scale_rows
|
||||
ng = scale_rows.shape[1]
|
||||
return (w.view(n, ng, hidden // ng) * scale_rows.unsqueeze(-1)).view(n, hidden)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Triton dequant kernel requires CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize("num_bits", [2, 4, 8])
|
||||
@pytest.mark.parametrize("group_size", [0, 256]) # 0 -> channel
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_ids", [1, 17, 4096])
|
||||
def test_dequant_gather(num_bits, group_size, dtype, num_ids):
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
vocab, hidden = 1000, 2048
|
||||
pack_factor = 32 // num_bits
|
||||
|
||||
# Random full-range int32 packed weights (covers the sign bit -> exercises the
|
||||
# arithmetic-shift + mask unpack path).
|
||||
weight_packed = torch.randint(
|
||||
-(2**31),
|
||||
2**31,
|
||||
(vocab, hidden // pack_factor),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
num_groups = 1 if group_size == 0 else hidden // group_size
|
||||
weight_scale = torch.rand(vocab, num_groups, dtype=dtype, device=device) + 0.01
|
||||
|
||||
ids = torch.randint(0, vocab, (num_ids,), dtype=torch.long, device=device)
|
||||
|
||||
out = _dequant_gather_triton(ids, weight_packed, weight_scale, hidden, num_bits)
|
||||
ref = _dequant_gather_torch(ids, weight_packed, weight_scale, hidden, num_bits)
|
||||
|
||||
assert out.shape == (num_ids, hidden)
|
||||
assert out.dtype == dtype
|
||||
torch.testing.assert_close(out, ref)
|
||||
@@ -1,509 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compile-guard tests for the ROCm RDNA3 W4A16 kernels (dense + MoE).
|
||||
|
||||
Verifies that the gfx1100 compilation and dispatch guards are hermetic:
|
||||
- On gfx1100: all ops exist, dispatch selects RDNA3 kernels.
|
||||
- On CDNA (gfx942/gfx950) or other non-gfx1100: ops must NOT exist,
|
||||
dispatch must fall through to Triton/Marlin, and no RDNA3 code
|
||||
path is reachable.
|
||||
|
||||
The negative (non-gfx1100) tests verify at three layers:
|
||||
1. Compile-level: on non-gfx1100 hardware, the RDNA3 ops are absent
|
||||
from the compiled _rocm_C extension — real binary verification.
|
||||
2. Static source analysis: parses CMakeLists.txt and torch_bindings.cpp
|
||||
to verify that all RDNA3 .cu files and op registrations are inside
|
||||
gfx1100-only guards.
|
||||
3. Runtime mock: patches on_gfx1100() to False and verifies that the
|
||||
Python dispatch chain rejects the RDNA3 path.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_compile_guards.py`.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
import vllm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 compile-guard tests are ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
kInt8Static,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not on_gfx1100(),
|
||||
reason="Requires gfx1100 hardware",
|
||||
)
|
||||
|
||||
not_gfx1100 = pytest.mark.skipif(
|
||||
on_gfx1100(),
|
||||
reason="This test verifies non-gfx1100 builds — skip on gfx1100",
|
||||
)
|
||||
|
||||
RDNA3_OPS = ["gptq_gemm_rdna3", "gptq_gemm_rdna3_wmma", "moe_gptq_gemm_rdna3"]
|
||||
RDNA3_CU_FILES = [
|
||||
"q_gemm_rdna3.cu",
|
||||
"q_gemm_rdna3_wmma.cu",
|
||||
"moe_q_gemm_rdna3.cu",
|
||||
]
|
||||
|
||||
|
||||
def _find_repo_root() -> Path | None:
|
||||
"""Walk up from this file to find the repo root (has CMakeLists.txt)."""
|
||||
for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]:
|
||||
if (parent / "CMakeLists.txt").exists() and (parent / "csrc").is_dir():
|
||||
return parent
|
||||
return None
|
||||
|
||||
|
||||
REPO_ROOT = _find_repo_root()
|
||||
|
||||
# Directory of the *installed* vllm python package. The .py guard checks read
|
||||
# from here so they verify the code that is actually imported at runtime — this
|
||||
# works even on CI images that ship the wheel instead of the python source tree
|
||||
# (where only csrc/ + CMakeLists.txt are checked out for building).
|
||||
VLLM_PKG_DIR: Path | None = (
|
||||
Path(vllm.__file__).parent if getattr(vllm, "__file__", None) else None
|
||||
)
|
||||
|
||||
needs_source = pytest.mark.skipif(
|
||||
REPO_ROOT is None,
|
||||
reason="C/CMake source tree not available (installed package only)",
|
||||
)
|
||||
|
||||
|
||||
def _read_source_or_skip(*relparts: str) -> str:
|
||||
"""Read a C/CMake source file from the repo tree, or skip if absent.
|
||||
|
||||
Used for csrc/ and CMakeLists.txt — these only exist in a source checkout,
|
||||
not in the installed wheel.
|
||||
"""
|
||||
assert REPO_ROOT is not None # callers are gated by @needs_source
|
||||
path = REPO_ROOT.joinpath(*relparts)
|
||||
if not path.exists():
|
||||
pytest.skip(f"{path} not present in this source tree")
|
||||
return path.read_text()
|
||||
|
||||
|
||||
def _read_pkg_source_or_skip(*relparts: str) -> str:
|
||||
"""Read a python source file from the installed vllm package.
|
||||
|
||||
Reflects the code actually loaded at runtime, so these guard checks run in
|
||||
CI against the wheel — no source checkout required. Only skips for an
|
||||
exotic install layout (namespace/zipimport) where __file__ is unavailable.
|
||||
"""
|
||||
if VLLM_PKG_DIR is None:
|
||||
pytest.skip("vllm package directory not resolvable (zip/namespace?)")
|
||||
assert VLLM_PKG_DIR is not None # narrow for mypy (skip above is NoReturn)
|
||||
path = VLLM_PKG_DIR.joinpath(*relparts)
|
||||
if not path.exists():
|
||||
pytest.skip(f"{path} not present in installed vllm package")
|
||||
return path.read_text()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part A: POSITIVE — on gfx1100, ops exist and dispatch works
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("op_name", RDNA3_OPS)
|
||||
def test_op_registered_on_gfx1100(op_name):
|
||||
"""On gfx1100, all RDNA3 ops must be registered in _rocm_C."""
|
||||
assert hasattr(torch.ops, "_rocm_C"), "_rocm_C module not loaded"
|
||||
assert hasattr(torch.ops._rocm_C, op_name), (
|
||||
f"_rocm_C.{op_name} not registered — "
|
||||
"check CMakeLists.txt VLLM_ROCM_HAS_GFX1100 "
|
||||
"and torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
def test_all_ops_present_or_all_absent():
|
||||
"""The 3 RDNA3 ops are behind the same #ifdef — all present or all absent.
|
||||
|
||||
Catches someone accidentally moving an op outside the guard.
|
||||
"""
|
||||
has_rocm_c = hasattr(torch.ops, "_rocm_C")
|
||||
if not has_rocm_c:
|
||||
pytest.skip("_rocm_C not loaded")
|
||||
|
||||
present = {op: hasattr(torch.ops._rocm_C, op) for op in RDNA3_OPS}
|
||||
values = set(present.values())
|
||||
assert len(values) == 1, (
|
||||
f"Guard inconsistency — some RDNA3 ops registered, others not: "
|
||||
f"{present}. Check torch_bindings.cpp #ifdef VLLM_ROCM_GFX1100 block."
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part B: NEGATIVE — compile-level verification on non-gfx1100
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
@pytest.mark.parametrize("op_name", RDNA3_OPS)
|
||||
def test_op_absent_on_non_gfx1100(op_name):
|
||||
"""On non-gfx1100 (CDNA), RDNA3 ops must NOT exist in _rocm_C.
|
||||
|
||||
This is the real compile-level check: the binary was built without
|
||||
gfx1100 support, so the ops should not have been compiled or registered.
|
||||
"""
|
||||
if not hasattr(torch.ops, "_rocm_C"):
|
||||
return
|
||||
assert not hasattr(torch.ops._rocm_C, op_name), (
|
||||
f"_rocm_C.{op_name} is registered on non-gfx1100 hardware — "
|
||||
"compile guard is broken: check CMakeLists.txt "
|
||||
"VLLM_ROCM_HAS_GFX1100 and torch_bindings.cpp #ifdef"
|
||||
)
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
def test_rocm_moe_not_supported_on_non_gfx1100():
|
||||
"""The RDNA3 MoE experts must not be selectable on non-gfx1100 hardware."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
Rdna3WNA16Experts,
|
||||
)
|
||||
|
||||
assert Rdna3WNA16Experts._supports_current_device() is False, (
|
||||
"Rdna3WNA16Experts reported support on non-gfx1100 — dispatch guard is broken"
|
||||
)
|
||||
|
||||
|
||||
@not_gfx1100
|
||||
def test_dense_kernel_rejects_on_non_gfx1100():
|
||||
"""RDNA3W4A16LinearKernel.can_implement must reject on non-gfx1100."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 dense kernel accepted on non-gfx1100: {reason}"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part C: Static source analysis (build-level guards)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@needs_source
|
||||
class TestCMakeGuards:
|
||||
"""Verify CMakeLists.txt only compiles RDNA3 .cu files for gfx1100."""
|
||||
|
||||
@staticmethod
|
||||
def _read_cmake():
|
||||
return _read_source_or_skip("CMakeLists.txt")
|
||||
|
||||
def test_rdna3_cu_files_inside_gfx1100_conditional(self):
|
||||
"""All RDNA3 .cu files must be listed inside the
|
||||
``if(VLLM_GPU_ARCHES MATCHES "gfx1100")`` block, not unconditionally.
|
||||
"""
|
||||
cmake = self._read_cmake()
|
||||
for cu_file in RDNA3_CU_FILES:
|
||||
assert cu_file in cmake, f"{cu_file} not found in CMakeLists.txt"
|
||||
|
||||
lines = cmake.splitlines()
|
||||
in_gfx1100_block = False
|
||||
for line in lines:
|
||||
if 'VLLM_GPU_ARCHES MATCHES "gfx1100"' in line:
|
||||
in_gfx1100_block = True
|
||||
if in_gfx1100_block and "endif()" in line:
|
||||
in_gfx1100_block = False
|
||||
if cu_file in line:
|
||||
assert in_gfx1100_block, (
|
||||
f"{cu_file} is listed OUTSIDE the gfx1100 "
|
||||
f"conditional in CMakeLists.txt — CDNA builds "
|
||||
f"would compile RDNA3 code. Line: {line.strip()}"
|
||||
)
|
||||
|
||||
def test_compile_definition_only_for_gfx1100(self):
|
||||
"""VLLM_ROCM_GFX1100 compile definition must be conditional."""
|
||||
cmake = self._read_cmake()
|
||||
lines = cmake.splitlines()
|
||||
in_gfx1100_block = False
|
||||
for line in lines:
|
||||
if "VLLM_ROCM_HAS_GFX1100)" in line:
|
||||
in_gfx1100_block = True
|
||||
if in_gfx1100_block and "endif()" in line:
|
||||
in_gfx1100_block = False
|
||||
if "VLLM_ROCM_GFX1100" in line and "target_compile_definitions" in line:
|
||||
assert in_gfx1100_block, (
|
||||
"VLLM_ROCM_GFX1100 compile definition is set outside "
|
||||
"the VLLM_ROCM_HAS_GFX1100 conditional — CDNA builds "
|
||||
f"would define it. Line: {line.strip()}"
|
||||
)
|
||||
|
||||
|
||||
@needs_source
|
||||
class TestTorchBindingsGuards:
|
||||
"""Verify torch_bindings.cpp gates all RDNA3 ops behind #ifdef."""
|
||||
|
||||
@staticmethod
|
||||
def _read_bindings():
|
||||
return _read_source_or_skip("csrc", "rocm", "torch_bindings.cpp")
|
||||
|
||||
def test_all_rdna3_ops_inside_ifdef(self):
|
||||
"""Every rdna3 op def/impl must be between #ifdef VLLM_ROCM_GFX1100
|
||||
and #endif. If any is outside, a CDNA build would try to register
|
||||
the op and link a symbol that doesn't exist.
|
||||
"""
|
||||
src = self._read_bindings()
|
||||
lines = src.splitlines()
|
||||
|
||||
inside_guard = False
|
||||
rdna3_lines_outside = []
|
||||
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "#ifdef VLLM_ROCM_GFX1100" in line:
|
||||
inside_guard = True
|
||||
elif line.strip() == "#endif" and inside_guard:
|
||||
inside_guard = False
|
||||
|
||||
if (
|
||||
"rdna3" in line.lower()
|
||||
and not line.strip().startswith("//")
|
||||
and not inside_guard
|
||||
):
|
||||
rdna3_lines_outside.append((i, line.strip()))
|
||||
|
||||
assert not rdna3_lines_outside, (
|
||||
"RDNA3 op references found OUTSIDE #ifdef VLLM_ROCM_GFX1100 "
|
||||
"in torch_bindings.cpp — these would break CDNA builds:\n"
|
||||
+ "\n".join(f" L{n}: {s}" for n, s in rdna3_lines_outside)
|
||||
)
|
||||
|
||||
def test_no_unconditional_rdna3_includes(self):
|
||||
"""No #include of RDNA3-specific headers outside the guard."""
|
||||
src = self._read_bindings()
|
||||
lines = src.splitlines()
|
||||
|
||||
inside_guard = False
|
||||
for i, line in enumerate(lines, 1):
|
||||
if "#ifdef VLLM_ROCM_GFX1100" in line:
|
||||
inside_guard = True
|
||||
elif line.strip() == "#endif" and inside_guard:
|
||||
inside_guard = False
|
||||
|
||||
if "#include" in line and "rdna3" in line.lower():
|
||||
assert inside_guard, (
|
||||
f"L{i}: RDNA3 include outside gfx1100 guard: {line.strip()}"
|
||||
)
|
||||
|
||||
|
||||
class TestCustomOpsGuards:
|
||||
"""Verify _custom_ops.py gates register_fake behind hasattr checks."""
|
||||
|
||||
@staticmethod
|
||||
def _read_custom_ops():
|
||||
return _read_pkg_source_or_skip("_custom_ops.py")
|
||||
|
||||
def test_register_fake_guarded_by_hasattr(self):
|
||||
"""Every register_fake for an RDNA3 op must be preceded by a hasattr
|
||||
check — otherwise it would crash on import on CDNA where the ops
|
||||
don't exist.
|
||||
"""
|
||||
src = self._read_custom_ops()
|
||||
for op in RDNA3_OPS:
|
||||
pattern = rf'register_fake\(\s*"_rocm_C::{op}"\s*\)'
|
||||
match = re.search(pattern, src)
|
||||
if match is None:
|
||||
continue
|
||||
preceding = src[: match.start()]
|
||||
last_hasattr = preceding.rfind(f'hasattr(torch.ops._rocm_C, "{op}")')
|
||||
assert last_hasattr != -1, (
|
||||
f'register_fake("_rocm_C::{op}") is not preceded by a '
|
||||
f"hasattr check — would crash on CDNA import"
|
||||
)
|
||||
gap = preceding[last_hasattr:].count("\n")
|
||||
assert gap <= 5, (
|
||||
f"hasattr guard for {op} is {gap} lines before "
|
||||
f"register_fake — suspiciously far; verify it's the "
|
||||
f"actual guard and not a coincidence"
|
||||
)
|
||||
|
||||
def test_no_toplevel_rocm_c_import(self):
|
||||
"""No top-level ``from vllm._rocm_C import`` — would crash on CDNA."""
|
||||
src = self._read_custom_ops()
|
||||
for line in src.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("#") or stripped.startswith("//"):
|
||||
continue
|
||||
assert "from vllm._rocm_C import" not in stripped, (
|
||||
f"Top-level import of _rocm_C in _custom_ops.py would "
|
||||
f"crash on CDNA: {stripped}"
|
||||
)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Part D: Runtime mock (simulate CDNA on gfx1100 hardware)
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestMoEDispatchMocked:
|
||||
"""Mock on_gfx1100() to False and verify RDNA3 MoE is unreachable."""
|
||||
|
||||
def test_kernel_unavailable_when_mocked_cdna(self):
|
||||
"""The device gate must reject when not on gfx1100."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
rdna3_moe_kernel_available,
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.rocm.on_gfx1100", return_value=False):
|
||||
assert rdna3_moe_kernel_available() is False
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_key",
|
||||
[kInt8Static, kInt4StaticAsym, kInt4Static32Asym, None],
|
||||
)
|
||||
def test_quant_scheme_rejects_non_symmetric_int4(self, weight_key):
|
||||
"""Only symmetric int4 weight-only schemes reach the kernel."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
Rdna3WNA16Experts,
|
||||
)
|
||||
|
||||
assert Rdna3WNA16Experts._supports_quant_scheme(weight_key, None) is False
|
||||
|
||||
@pytest.mark.parametrize("weight_key", [kInt4Static, kInt4Static32])
|
||||
def test_quant_scheme_accepts_symmetric_int4(self, weight_key):
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
Rdna3WNA16Experts,
|
||||
)
|
||||
|
||||
assert Rdna3WNA16Experts._supports_quant_scheme(weight_key, None) is True
|
||||
|
||||
def test_kernel_unavailable_when_op_missing(self):
|
||||
"""The device gate returns False when the C++ op doesn't exist."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
rdna3_moe_kernel_available,
|
||||
)
|
||||
|
||||
fake_rocm_c = type("FakeRocmC", (), {"gptq_gemm_rdna3": None})()
|
||||
with patch.object(torch, "ops", create=True) as mock_ops:
|
||||
mock_ops._rocm_C = fake_rocm_c
|
||||
assert rdna3_moe_kernel_available() is False
|
||||
|
||||
def test_kernel_unavailable_when_rocm_c_absent(self):
|
||||
"""The device gate returns False when _rocm_C doesn't exist at all."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
rdna3_moe_kernel_available,
|
||||
)
|
||||
|
||||
fake_ops = type("FakeOps", (), {})()
|
||||
with patch.object(torch, "ops", fake_ops):
|
||||
assert rdna3_moe_kernel_available() is False
|
||||
|
||||
|
||||
class TestDenseKernelSelectionMocked:
|
||||
"""Mock on_gfx1100() and verify dense RDNA3 kernel is not selected."""
|
||||
|
||||
@gfx1100_only
|
||||
def test_can_implement_rejects_when_mocked_cdna(self):
|
||||
"""RDNA3W4A16LinearKernel.can_implement must reject on mocked CDNA."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E501
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
ok, _ = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is True
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
patch("vllm.platforms.rocm._ON_GFX1100", False),
|
||||
):
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is False, f"RDNA3 kernel accepted on simulated CDNA: {reason}"
|
||||
|
||||
@gfx1100_only
|
||||
def test_chooser_skips_rdna3_when_mocked_cdna(self):
|
||||
"""choose_mp_linear_kernel must NOT return RDNA3 on mocked CDNA."""
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E501
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
with (
|
||||
patch("vllm.platforms.rocm.on_gfx1100", return_value=False),
|
||||
patch("vllm.platforms.rocm._ON_GFX1100", False),
|
||||
):
|
||||
chosen = choose_mp_linear_kernel(config)
|
||||
assert chosen.__name__ != "RDNA3W4A16LinearKernel", (
|
||||
"RDNA3 kernel was selected on simulated CDNA — "
|
||||
"choose_mp_linear_kernel guard is broken"
|
||||
)
|
||||
|
||||
|
||||
class TestWNA16OracleWiring:
|
||||
"""The RDNA3 backend must reach the kernel only through the oracle."""
|
||||
|
||||
def test_backend_maps_to_rdna3_experts(self):
|
||||
from vllm.model_executor.layers.fused_moe.experts.rdna3_moe import (
|
||||
Rdna3WNA16Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
backend_to_kernel_cls,
|
||||
map_wna16_backend,
|
||||
)
|
||||
|
||||
assert backend_to_kernel_cls(WNA16MoEBackend.RDNA3) == [Rdna3WNA16Experts]
|
||||
assert map_wna16_backend("rdna3") == WNA16MoEBackend.RDNA3
|
||||
|
||||
def test_backend_is_offered_before_the_triton_fallback(self):
|
||||
"""Priority order: the native kernel outranks Triton when supported."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import (
|
||||
WNA16MoEBackend,
|
||||
_get_priority_backends,
|
||||
)
|
||||
|
||||
backends = _get_priority_backends()
|
||||
assert backends.index(WNA16MoEBackend.RDNA3) < backends.index(
|
||||
WNA16MoEBackend.TRITON
|
||||
)
|
||||
@@ -1,442 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the ROCm RDNA3 fused MoE W4A16 HIP kernel (gfx1100).
|
||||
|
||||
Tests ``moe_gptq_gemm_rdna3`` against the dense ``gptq_gemm_rdna3`` as
|
||||
reference: builds RDNA3-format weights (shuffled int32, synthesized qzeros),
|
||||
runs the fused MoE kernel, and compares per-expert results.
|
||||
|
||||
Model parameters taken from:
|
||||
- cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit
|
||||
(hidden=2048, inter=768, E=128, top_k=8, G=32)
|
||||
- Qwen3.6-35B-A3B-GPTQ-W4A16-G32
|
||||
(hidden=2048, inter=512, E=256, top_k=8, G=32)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_moe_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 MoE W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm import _custom_ops as ops # noqa: E402
|
||||
from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( # noqa: E402
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( # noqa: E402
|
||||
_process_weights_rdna3,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "moe_gptq_gemm_rdna3")
|
||||
),
|
||||
reason="Requires gfx1100 with moe_gptq_gemm_rdna3 op",
|
||||
)
|
||||
|
||||
# Model configurations: real K/N/top_k/group_size dims, E capped at 16 to
|
||||
# fit in test GPU memory (full E=128/256 would need >20GB for weights alone).
|
||||
# Kernel behavior is E-independent (per-expert tiling), so E=16 is sufficient.
|
||||
MODEL_CONFIGS = [
|
||||
# cyankiwi/Qwen3-30B-A3B-Instruct-2507-AWQ-4bit dims (E capped)
|
||||
pytest.param(16, 2048, 768, 8, 32, id="Qwen3-30B-A3B"),
|
||||
# Qwen3.6-35B-A3B-GPTQ-W4A16-G32 dims (E capped)
|
||||
pytest.param(16, 2048, 512, 8, 32, id="Qwen3.6-35B-A3B"),
|
||||
]
|
||||
|
||||
# Token counts: decode (1), small batch (4), medium (16), prefill (64)
|
||||
NUM_TOKENS = [1, 4, 16, 64, 256, 512]
|
||||
|
||||
|
||||
def _make_packed_weights(E, K, N):
|
||||
"""Create random 4-bit packed weights [E, K/8, N] int32 + shuffle."""
|
||||
w = torch.randint(0, 16, (E, K, N), dtype=torch.int32, device=device)
|
||||
packed = torch.zeros(E, K // 8, N, dtype=torch.int32, device=device)
|
||||
for i in range(8):
|
||||
packed |= (w[:, i::8, :] & 0xF) << (i * 4)
|
||||
for e in range(E):
|
||||
we = packed[e].contiguous()
|
||||
ops.gptq_shuffle(we, 4)
|
||||
packed[e] = we
|
||||
return packed
|
||||
|
||||
|
||||
def _make_scales(E, groups, N, dtype):
|
||||
return torch.rand(E, groups, N, dtype=dtype, device=device) * 0.1
|
||||
|
||||
|
||||
def _make_qzeros(E, groups, N):
|
||||
zeros = torch.full(
|
||||
(groups, N),
|
||||
scalar_types.uint4b8.bias - 1,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
qz = pack_quantized_values_into_int32(
|
||||
zeros,
|
||||
scalar_types.uint4b8,
|
||||
packed_dim=1,
|
||||
)
|
||||
return qz.unsqueeze(0).expand(E, -1, -1).contiguous()
|
||||
|
||||
|
||||
def _make_ct_weights(E, N, K, group_size, dtype):
|
||||
"""Compressed-tensors canonical N-first layout: packed ``[E, N, K // 8]``
|
||||
and scales ``[E, N, K // group_size]``, plus the dequantized weights."""
|
||||
q = torch.randint(0, 16, (E, N, K), dtype=torch.int32, device=device)
|
||||
s = torch.rand(E, N, K // group_size, dtype=dtype, device=device) * 0.1 + 0.01
|
||||
packed = pack_quantized_values_into_int32(q, scalar_types.uint4b8, packed_dim=2)
|
||||
w = (q - 8).float() * s.float().repeat_interleave(group_size, dim=2)
|
||||
return packed.contiguous(), s, w
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("block_size_m", [1, 4])
|
||||
def test_fused_moe_w1_matches_dense(
|
||||
E, K, N_inter, top_k, group_size, M, dtype, block_size_m
|
||||
):
|
||||
"""w1 GEMM via fused kernel matches per-expert dense kernel."""
|
||||
N_gate_up = N_inter * 2
|
||||
groups = K // group_size
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, groups, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, groups, N_gate_up)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, block_size_m, E)
|
||||
|
||||
# Fused kernel
|
||||
fused_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
fused_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
block_size_m,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
|
||||
# Per-expert dense reference
|
||||
ref_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
for m in range(M):
|
||||
for k in range(top_k):
|
||||
e = topk_ids[m, k].item()
|
||||
flat = m * top_k + k
|
||||
ref = ops.gptq_gemm_rdna3(
|
||||
x[m : m + 1],
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
False,
|
||||
)
|
||||
ref_out[flat] = ref.squeeze()
|
||||
|
||||
# Split-K atomics can cause minor fp16/bf16 rounding differences
|
||||
# at large K (e.g. K=2048 → 8 K-blocks). Use allclose, not equal.
|
||||
atol = 0.5 if dtype == torch.bfloat16 else 0.1
|
||||
assert torch.allclose(fused_out, ref_out, atol=atol, rtol=0.01), (
|
||||
f"max diff: {(fused_out - ref_out).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_fused_moe_output_topk_reduces(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
"""output_topk fuses moe_sum: multiple experts write to same output row."""
|
||||
groups = K // group_size
|
||||
|
||||
torch.manual_seed(123)
|
||||
x = torch.randn(M * top_k, K, dtype=dtype, device=device)
|
||||
w = _make_packed_weights(E, K, N_inter)
|
||||
ws = _make_scales(E, groups, N_inter, dtype)
|
||||
wz = _make_qzeros(E, groups, N_inter)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
torch.randn(M, top_k, device=device),
|
||||
dim=-1,
|
||||
).float()
|
||||
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
# Without output_topk: write to [M*top_k, N] then moe_sum
|
||||
flat_out = torch.zeros(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
flat_out,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
0,
|
||||
)
|
||||
ref = torch.zeros(M, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_sum(flat_out.view(M, top_k, N_inter), ref)
|
||||
|
||||
# With output_topk: write directly to [M, N]
|
||||
fused = torch.zeros(M, N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
fused,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
atol = 1.0 if dtype == torch.bfloat16 else 0.1
|
||||
assert torch.allclose(fused, ref, atol=atol, rtol=0.01), (
|
||||
f"max diff: {(fused - ref).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_full_moe_e2e(E, K, N_inter, top_k, group_size, M, dtype):
|
||||
"""Full MoE forward: w1 + silu_and_mul + w2 with output_topk reduce."""
|
||||
N_gate_up = N_inter * 2
|
||||
hidden = K
|
||||
|
||||
torch.manual_seed(7)
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
w13 = _make_packed_weights(E, K, N_gate_up)
|
||||
w13_s = _make_scales(E, K // group_size, N_gate_up, dtype)
|
||||
w13_z = _make_qzeros(E, K // group_size, N_gate_up)
|
||||
w2 = _make_packed_weights(E, N_inter, hidden)
|
||||
w2_s = _make_scales(E, N_inter // group_size, hidden, dtype)
|
||||
w2_z = _make_qzeros(E, N_inter // group_size, hidden)
|
||||
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(
|
||||
torch.randn(M, top_k, device=device),
|
||||
dim=-1,
|
||||
).float()
|
||||
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
# Fused path (what apply() does)
|
||||
w1_out = torch.zeros(M * top_k, N_gate_up, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
w1_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
|
||||
fused = torch.zeros(M, hidden, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
act_out,
|
||||
fused,
|
||||
w2,
|
||||
w2_s,
|
||||
w2_z,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
# Per-expert reference
|
||||
ref = torch.zeros(M, hidden, dtype=dtype, device=device)
|
||||
for m_idx in range(M):
|
||||
for k_idx in range(top_k):
|
||||
e = topk_ids[m_idx, k_idx].item()
|
||||
w = topk_w[m_idx, k_idx].item()
|
||||
r1 = ops.gptq_gemm_rdna3(
|
||||
x[m_idx : m_idx + 1],
|
||||
w13[e],
|
||||
w13_z[e],
|
||||
w13_s[e],
|
||||
False,
|
||||
)
|
||||
a = torch.empty(1, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, a, r1)
|
||||
r2 = ops.gptq_gemm_rdna3(
|
||||
a,
|
||||
w2[e],
|
||||
w2_z[e],
|
||||
w2_s[e],
|
||||
False,
|
||||
)
|
||||
ref[m_idx] += r2.squeeze() * w
|
||||
|
||||
# E2E chains w1 + activation + w2 + topk_w + output_topk reduce.
|
||||
# Each step accumulates rounding error (split-K atomics, topk_w
|
||||
# multiply order). Use relative L2 norm like the dense kernel test.
|
||||
diff_l2 = torch.norm(fused.float() - ref.float())
|
||||
ref_l2 = torch.norm(ref.float())
|
||||
rel_l2 = (diff_l2 / ref_l2).item() if ref_l2 > 0 else 0.0
|
||||
threshold = 0.05 if dtype == torch.float16 else 0.10
|
||||
assert rel_l2 < threshold, (
|
||||
f"rel L2 = {rel_l2:.4f} (threshold {threshold}), "
|
||||
f"max abs diff: {(fused - ref).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
def test_expert_id_minus_one():
|
||||
"""Kernel handles expert_id == -1 (expert parallelism) without crash."""
|
||||
# Qwen3-30B-A3B dims (E capped for memory)
|
||||
E, K, N = 16, 2048, 768
|
||||
groups = K // 32
|
||||
|
||||
w = _make_packed_weights(E, K, N)
|
||||
ws = _make_scales(E, groups, N, torch.bfloat16)
|
||||
wz = _make_qzeros(E, groups, N)
|
||||
x = torch.randn(1, K, dtype=torch.bfloat16, device=device)
|
||||
|
||||
# Manually create sorted_token_ids/expert_ids with -1
|
||||
sorted_ids = torch.tensor([0], dtype=torch.int32, device=device)
|
||||
expert_ids = torch.tensor([-1], dtype=torch.int32, device=device)
|
||||
ntp = torch.tensor([1], dtype=torch.int32, device=device)
|
||||
|
||||
out = torch.zeros(1, N, dtype=torch.bfloat16, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
out,
|
||||
w,
|
||||
ws,
|
||||
wz,
|
||||
torch.empty(0, device=device),
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
|
||||
# Output should remain zero (expert skipped)
|
||||
assert torch.equal(out, torch.zeros_like(out))
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_process_weights_from_compressed_tensors_layout(dtype):
|
||||
"""Weights in the layout compressed-tensors hands to the oracle go through
|
||||
the RDNA3 post-processing and a full MoE forward, against a dequantized
|
||||
reference. The tests above build the kernel layout directly, so they do
|
||||
not notice when the layout upstream of the kernel changes (#52798)."""
|
||||
E, K, N_inter, top_k, group_size, M = 8, 512, 256, 2, 32, 16
|
||||
torch.manual_seed(5)
|
||||
w13, w13_s, w13_ref = _make_ct_weights(E, 2 * N_inter, K, group_size, dtype)
|
||||
w2, w2_s, w2_ref = _make_ct_weights(E, K, N_inter, group_size, dtype)
|
||||
w13, w2, w13_s, w2_s, w13_z, w2_z, *_ = _process_weights_rdna3(
|
||||
w13, w2, w13_s, w2_s, group_size
|
||||
)
|
||||
assert w13_z is not None and w2_z is not None
|
||||
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(torch.randn(M, top_k, device=device), dim=-1).float()
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
w1_out = torch.zeros(M * top_k, 2 * N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
w1_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
|
||||
out = torch.zeros(M, K, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
act_out,
|
||||
out,
|
||||
w2,
|
||||
w2_s,
|
||||
w2_z,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
ref = torch.zeros(M, K, device=device)
|
||||
for m in range(M):
|
||||
for k in range(top_k):
|
||||
e = int(topk_ids[m, k])
|
||||
h = w13_ref[e] @ x[m].float()
|
||||
a = F.silu(h[:N_inter]) * h[N_inter:]
|
||||
ref[m] += (w2_ref[e] @ a) * topk_w[m, k]
|
||||
rel_l2 = (torch.norm(out.float() - ref) / torch.norm(ref)).item()
|
||||
assert rel_l2 < (0.02 if dtype == torch.float16 else 0.05), rel_l2
|
||||
@@ -1,276 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the ROCm RDNA3 W4A16 GPTQ kernel (gfx1100).
|
||||
|
||||
Exercises ``RDNA3W4A16LinearKernel`` end-to-end: it builds a layer with
|
||||
GPTQ-format checkpoint parameters, runs ``process_weights_after_loading``
|
||||
(weight shuffle + zero-point synthesis), then ``apply_weights``, and compares
|
||||
the result against an fp32 reference dequant-and-matmul.
|
||||
|
||||
The kernel is exposed via ``torch.ops._rocm_C.gptq_gemm_rdna3`` and is only
|
||||
built for gfx11; tests are skipped elsewhere.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import ( # noqa: E402
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
from vllm.utils.torch_utils import set_random_seed # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
WEIGHT_TYPE = scalar_types.uint4b8 # symmetric int4, bias = 8
|
||||
PACK_FACTOR = 8 # 8 x 4-bit nibbles per int32
|
||||
|
||||
# Skip everything in this module unless we are on the only architecture the
|
||||
# kernel is built/registered for.
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
|
||||
),
|
||||
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reference implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _reference(
|
||||
x_mk: torch.Tensor,
|
||||
q_int4_kn: torch.Tensor,
|
||||
scales_gn: torch.Tensor,
|
||||
zeros_gn: torch.Tensor | None,
|
||||
group_size: int,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""fp32 reference for the RDNA3 W4A16 op.
|
||||
|
||||
x_mk: [M, K] fp16/bf16 activations.
|
||||
q_int4_kn: [K, N] int32 raw stored nibbles in [0, 15].
|
||||
scales_gn: [K//G, N] per-group scales (act dtype).
|
||||
zeros_gn: [K//G, N] int32 raw stored zero points in [0, 15], or None
|
||||
for the symmetric path (kernel synthesizes stored zero = 7).
|
||||
group_size: G.
|
||||
|
||||
The kernel applies the GPTQv1 "+1" zero-point quirk, so the effective
|
||||
zero is ``stored_zero + 1`` (symmetric path: 7 + 1 == bias == 8).
|
||||
"""
|
||||
K, N = q_int4_kn.shape
|
||||
s_full = scales_gn.repeat_interleave(group_size, dim=0).to(torch.float32) # [K,N]
|
||||
if zeros_gn is None:
|
||||
z_full = torch.full(
|
||||
(K, N), float(WEIGHT_TYPE.bias), device=x_mk.device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
z_full = (zeros_gn + 1).repeat_interleave(group_size, dim=0).to(torch.float32)
|
||||
w_fp = (q_int4_kn.to(torch.float32) - z_full) * s_full # [K, N]
|
||||
out = x_mk.to(torch.float32) @ w_fp # [M, N]
|
||||
if bias is not None:
|
||||
out = out + bias.to(torch.float32)
|
||||
return out.to(x_mk.dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Layer construction (GPTQ checkpoint format)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_layer(
|
||||
q_int4_kn: torch.Tensor,
|
||||
scales_gn: torch.Tensor,
|
||||
zeros_gn: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.nn.Module:
|
||||
"""Build a dummy layer carrying GPTQ-format params, as the loader would."""
|
||||
no_loader = lambda *args, **kwargs: None # noqa: E731
|
||||
|
||||
# qweight: int4 packed along K into int32 -> [K//8, N].
|
||||
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"qweight",
|
||||
PackedvLLMParameter(
|
||||
data=qweight,
|
||||
weight_loader=no_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=PACK_FACTOR,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"scales",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_gn.to(dtype),
|
||||
weight_loader=no_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
),
|
||||
)
|
||||
if zeros_gn is not None:
|
||||
# qzeros: int4 packed along N into int32 -> [K//G, N//8].
|
||||
qzeros = pack_quantized_values_into_int32(zeros_gn, WEIGHT_TYPE, packed_dim=1)
|
||||
layer.register_parameter(
|
||||
"qzeros",
|
||||
PackedvLLMParameter(
|
||||
data=qzeros,
|
||||
weight_loader=no_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=1,
|
||||
packed_factor=PACK_FACTOR,
|
||||
),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def _run_kernel(
|
||||
x_mk: torch.Tensor,
|
||||
q_int4_kn: torch.Tensor,
|
||||
scales_gn: torch.Tensor,
|
||||
zeros_gn: torch.Tensor | None,
|
||||
group_size: int,
|
||||
bias: torch.Tensor | None,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
K, N = q_int4_kn.shape
|
||||
has_zp = zeros_gn is not None
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=WEIGHT_TYPE,
|
||||
act_type=dtype,
|
||||
group_size=group_size,
|
||||
zero_points=has_zp,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok, f"can_implement rejected a supported config: {reason}"
|
||||
|
||||
layer = _build_layer(q_int4_kn, scales_gn, zeros_gn, dtype)
|
||||
kernel = RDNA3W4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros" if has_zp else None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel.apply_weights(layer, x_mk, bias=bias)
|
||||
|
||||
|
||||
# Relative-L2 tolerance per dtype. The bf16 path widens dequantized weights
|
||||
# to fp32 and accumulates in fp32, so it matches the reference almost exactly
|
||||
# (<0.4% incl. the WMMA prefill path). The fp16 path uses the exllamav2
|
||||
# "+1024" bit-trick (see qdq_4_rdna3.cuh): the dequantized weight is recovered
|
||||
# as the fp16 difference of two ~1024*scale magnitudes, which sheds low-order
|
||||
# mantissa bits and leaves ~2-3% relative noise that accumulates over K. We
|
||||
# compare on the relative Frobenius norm rather than elementwise, since the
|
||||
# bit-trick noise produces large *relative* errors on individual near-zero
|
||||
# outputs that carry negligible absolute weight.
|
||||
_REL_L2_TOL = {torch.float16: 5e-2, torch.bfloat16: 1e-2}
|
||||
|
||||
|
||||
def _assert_close(out: torch.Tensor, ref: torch.Tensor, dtype: torch.dtype):
|
||||
rel_l2 = (out.to(torch.float32) - ref.to(torch.float32)).norm() / ref.to(
|
||||
torch.float32
|
||||
).norm()
|
||||
tol = _REL_L2_TOL[dtype]
|
||||
assert rel_l2 < tol, f"relative L2 error {rel_l2:.4f} exceeds {tol} for {dtype}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Forward correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# (M, K, N, group_size). M spans the scalar decode path (small M) and the
|
||||
# WMMA prefill path (M >= 16 on the bf16 dispatch). K/N satisfy the kernel's
|
||||
# divisibility constraints (K % G == 0, K % 8 == 0, N % 8 == 0).
|
||||
MKNG_SHAPES = [
|
||||
(1, 128, 128, 128), # single group, decode
|
||||
(2, 256, 256, 128), # two groups
|
||||
(8, 256, 512, 64), # M=8 scalar, smaller group
|
||||
(16, 512, 256, 128), # M=16 -> WMMA path for bf16
|
||||
(32, 512, 512, 64), # larger prefill
|
||||
]
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True], ids=["no_zp", "with_zp"])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G", MKNG_SHAPES, ids=[f"m{m}_k{k}_n{n}_g{g}" for m, k, n, g in MKNG_SHAPES]
|
||||
)
|
||||
def test_rdna3_w4a16_matches_reference(dtype, has_zp, M, K, N, G, dist_init):
|
||||
set_random_seed(0)
|
||||
assert K % G == 0 and K % PACK_FACTOR == 0 and N % PACK_FACTOR == 0
|
||||
|
||||
groups = K // G
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
q_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
scales_gn = (
|
||||
0.05 * torch.rand((groups, N), device=device, dtype=torch.float32) + 0.01
|
||||
).to(dtype)
|
||||
zeros_gn = (
|
||||
torch.randint(0, 16, (groups, N), device=device, dtype=torch.int32)
|
||||
if has_zp
|
||||
else None
|
||||
)
|
||||
|
||||
out = _run_kernel(x_mk, q_int4_kn, scales_gn, zeros_gn, G, None, dtype)
|
||||
ref = _reference(x_mk, q_int4_kn, scales_gn, zeros_gn, G, None)
|
||||
|
||||
assert out.shape == (M, N) and out.dtype == dtype
|
||||
_assert_close(out, ref, dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("M", [1, 32], ids=["decode", "prefill"])
|
||||
def test_rdna3_w4a16_bias(dtype, M, dist_init):
|
||||
"""Bias is added on both the scalar (M=1) and WMMA (M=32) paths."""
|
||||
set_random_seed(0)
|
||||
K, N, G = 512, 256, 128
|
||||
groups = K // G
|
||||
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
q_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
scales_gn = (
|
||||
0.05 * torch.rand((groups, N), device=device, dtype=torch.float32) + 0.01
|
||||
).to(dtype)
|
||||
bias = (0.1 * torch.randn(N, device=device, dtype=torch.float32)).to(dtype)
|
||||
|
||||
out = _run_kernel(x_mk, q_int4_kn, scales_gn, None, G, bias, dtype)
|
||||
ref = _reference(x_mk, q_int4_kn, scales_gn, None, G, bias)
|
||||
|
||||
_assert_close(out, ref, dtype)
|
||||
@@ -1,585 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Determinism + FP32-reference correctness tests for the RDNA3 W4A16 GEMM.
|
||||
|
||||
The split-K epilogues of ``gptq_gemm_rdna3`` (scalar path) and the WMMA
|
||||
path previously accumulated per-split low-precision partials directly into
|
||||
the output via CAS atomics. With multiple writers per output element the
|
||||
accumulation order — and therefore the rounded result — varied per call:
|
||||
repeated fixed-input calls produced a different output almost every time on
|
||||
gfx1100 once more than a few split blocks contended.
|
||||
|
||||
Both paths now store FP32 split partials and reduce them in a fixed order
|
||||
with a single final low-precision rounding, which is deterministic by
|
||||
construction. When the split count is 1 every path stores directly
|
||||
(single writer per output element).
|
||||
|
||||
These tests assert, through the public op path:
|
||||
|
||||
A. Repeatability — ``torch.equal`` across repeated identical calls.
|
||||
B. Numerical correctness — against an FP32 dequantized reference.
|
||||
|
||||
(B) exists because repeatability alone cannot see the failure mode the
|
||||
deterministic epilogue introduces: the FP32 partials scratch is
|
||||
``at::empty`` (its coverage invariant is documented at
|
||||
``alloc_wmma_partials`` in ``csrc/rocm/q_gemm_rdna3_wmma.cu``), and across
|
||||
repeated calls the caching allocator hands back the same block — a slot
|
||||
that ever went unwritten would be bitwise-repeatable *and* wrong.
|
||||
|
||||
Reference semantics (uint4b8, GPTQv1, synthesized zero points):
|
||||
W[k, n] = (q[k, n] - 8) * float(round_to_dtype(scale[g, n])), g = k//G
|
||||
ref[m, n] = x[m, :] @ W[:, n] (FP32 matmul)
|
||||
|
||||
Tolerances are derived from the kernel's rounding structure, not tuned to
|
||||
pass (measured max errors on W7900/gfx1100 in parentheses):
|
||||
|
||||
* scalar bf16 — dequant keeps FP32 precision end-to-end (magic-value
|
||||
v_dot2 path); the only rounding is the final cast. Measured max ~0.06
|
||||
on the tested shapes (|ref| up to ~30); tolerance 0.10 with headroom
|
||||
for larger |ref| at the final-cast ulp.
|
||||
* scalar fp16 — the dequant uses the exact factored form (magic values in
|
||||
fp16, (y, z) correction in fp32; see qdq_4_rdna3.cuh), so the only fp16
|
||||
roundings are the final output cast and the B-tile-free dot inputs:
|
||||
max_abs <= 0.5 * ulp_fp16(max|ref|) * 1.5 + 1e-3 (measured ~0.015 at
|
||||
|ref|~37; the pre-fix baked-constant form measured ~1.0-1.4 on the same
|
||||
shapes — 94-200x worse).
|
||||
* WMMA bf16 — B is narrowed to bf16 per cell (one rounding,
|
||||
<= 0.5*ulp_bf16(|B|)), accumulated in FP32; measured max ≈ 0.09 at
|
||||
K=4096 — tolerance 0.25.
|
||||
* WMMA fp16 — B narrowed to fp16 via the precise sub-then-mul variant
|
||||
(one rounding, ~1e-4); measured max ≈ 0.015 — tolerance 0.05.
|
||||
|
||||
All four are orders of magnitude below what an unwritten scratch slot
|
||||
(caching-allocator garbage / NaN) or an indexing bug (errors of the same
|
||||
scale as |ref| ≈ 7) would produce.
|
||||
|
||||
Test hygiene: the op path needs a single-process distributed group and a
|
||||
current VLLM config; tests use the repo-standard ``dist_init`` fixture from
|
||||
``tests/conftest.py`` (real context manager, temp-file rendezvous so no
|
||||
hardcoded MASTER_PORT can collide in parallel CI, and the suite's standard
|
||||
post-test cleanup).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
from vllm.model_executor.parameter import ( # noqa: E402
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
WEIGHT_TYPE = scalar_types.uint4b8
|
||||
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
|
||||
),
|
||||
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
|
||||
)
|
||||
|
||||
REPEATS = 20
|
||||
GROUP = 128
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
|
||||
# max_abs_error tolerances vs the FP32 reference, per (path, dtype).
|
||||
# Derivation in the module docstring; ~2-3x headroom over measured maxima.
|
||||
TOL = {
|
||||
("scalar", torch.bfloat16): 0.10,
|
||||
("scalar", torch.float16): 0.025,
|
||||
("wmma", torch.bfloat16): 0.25,
|
||||
("wmma", torch.float16): 0.05,
|
||||
}
|
||||
|
||||
|
||||
def _build_layer(k, n, seed, dtype, group=GROUP):
|
||||
torch.manual_seed(seed)
|
||||
q_int4_kn = torch.randint(0, 16, (k, n), dtype=torch.int32)
|
||||
scales_gn = (torch.randn(k // group, n) * 0.01 + 0.02).to(dtype)
|
||||
qweight = pack_quantized_values_into_int32(q_int4_kn, WEIGHT_TYPE, packed_dim=0)
|
||||
no_loader = lambda *a, **kw: None # noqa: E731
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"qweight",
|
||||
PackedvLLMParameter(
|
||||
data=qweight,
|
||||
weight_loader=no_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=8,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"scales",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_gn, weight_loader=no_loader, input_dim=0, output_dim=1
|
||||
),
|
||||
)
|
||||
layer.to(device)
|
||||
return layer, q_int4_kn, scales_gn
|
||||
|
||||
|
||||
def _prepare(layer, dtype, k, n, group=GROUP):
|
||||
cfg = MPLinearLayerConfig(
|
||||
full_weight_shape=(k, n),
|
||||
partition_weight_shape=(k, n),
|
||||
weight_type=WEIGHT_TYPE,
|
||||
act_type=dtype,
|
||||
group_size=group,
|
||||
zero_points=False,
|
||||
)
|
||||
kernel = RDNA3W4A16LinearKernel(
|
||||
cfg, w_q_param_name="qweight", w_s_param_name="scales", w_zp_param_name=None
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
return kernel._get_weight_params(layer)
|
||||
|
||||
|
||||
def _reference(m, k, dtype, seed, q_int4_kn, scales_gn, group=GROUP):
|
||||
torch.manual_seed(seed + 1)
|
||||
x = torch.randn(m, k, device=device, dtype=dtype)
|
||||
# FP32 reference; see module docstring for the dequant semantics.
|
||||
w_f32 = (q_int4_kn.to(device).float() - 8.0) * scales_gn.to(
|
||||
device
|
||||
).repeat_interleave(group, dim=0).float()
|
||||
return x, x.float() @ w_f32
|
||||
|
||||
|
||||
def _run_op(x, w_q, w_zp, w_s):
|
||||
return torch.ops._rocm_C.gptq_gemm_rdna3(x, w_q, w_zp, w_s, False)
|
||||
|
||||
|
||||
def _outputs_and_ref(M, K, N, seed, dtype, repeats=1, group=GROUP):
|
||||
layer, q_int4_kn, scales_gn = _build_layer(K, N, seed, dtype, group)
|
||||
w_q, w_s, w_zp = _prepare(layer, dtype, K, N, group)
|
||||
x, ref = _reference(M, K, dtype, seed, q_int4_kn, scales_gn, group)
|
||||
outs = [_run_op(x, w_q, w_zp, w_s) for _ in range(repeats)]
|
||||
return outs, ref
|
||||
|
||||
|
||||
def _path_for(dtype, m):
|
||||
"""Public dispatch: both dtypes reach WMMA at M>=12 (see the
|
||||
WMMA_MIN_M crossover table in csrc/rocm/q_gemm_rdna3.cu)."""
|
||||
if m >= 12:
|
||||
return "wmma"
|
||||
return "scalar"
|
||||
|
||||
|
||||
def _assert_repeatable(outs):
|
||||
for o in outs[1:]:
|
||||
assert torch.equal(o, outs[0]), (
|
||||
"RDNA3 W4A16 split-K produced differing outputs for identical inputs"
|
||||
)
|
||||
|
||||
|
||||
def _assert_close_to_ref(out, ref, path, dtype):
|
||||
err = (out.float() - ref).abs()
|
||||
max_abs = err.max().item()
|
||||
mean_abs = err.mean().item()
|
||||
tol = TOL[(path, dtype)]
|
||||
assert torch.isfinite(out.float()).all(), "non-finite output"
|
||||
assert max_abs <= tol, (
|
||||
f"RDNA3 W4A16 {path} output deviates from the FP32 dequantized "
|
||||
f"reference: max_abs={max_abs:.4f} > tol={tol} (mean_abs="
|
||||
f"{mean_abs:.4f}). This catches values that are bitwise-repeatable "
|
||||
"but wrong (e.g. an unwritten FP32 scratch slot)."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A. Repeatability (bit-exact across identical calls)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_splitk_bit_repeatable(dist_init, dtype):
|
||||
"""Scalar path at production-like contention: K=4096 splits into
|
||||
ceil(4096/256) = 16 concurrent writers per output element (the old CAS
|
||||
bug's onset was between 2 and 4 writers, so this sits well inside the
|
||||
failure regime rather than at the empirical threshold)."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=4096, N=4096, seed=1234, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_small_m_bit_repeatable(dist_init, dtype):
|
||||
"""Scalar M>1 tiles share the same deterministic epilogue."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=8, K=4096, N=4096, seed=1235, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_single_split_bit_repeatable(dist_init, dtype):
|
||||
"""z_count == 1 (K=256 < BLOCK_KN_SIZE): direct-store epilogue, no
|
||||
scratch, no reduce pass."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=256, N=4096, seed=1238, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_splitk_bit_repeatable(dist_init, dtype):
|
||||
"""WMMA path (both dtypes M >= 12), split-K active.
|
||||
|
||||
K=6656 gives K_SPLIT=4 under the upstream heuristic; the deterministic
|
||||
epilogue must be bit-repeatable regardless of the split count.
|
||||
"""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=m, K=6656, N=4096, seed=1236, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_large_m_bit_repeatable(dist_init, dtype):
|
||||
"""Large-M WMMA tiles (128x64 kernels) with the deterministic epilogue."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=128, K=6656, N=4096, seed=1237, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B. Numerical correctness against the FP32 dequantized reference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_matches_fp32_reference(dist_init, dtype):
|
||||
"""Scalar split-K (K=4096, 16 writers) vs the FP32 reference."""
|
||||
m = 1
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1240, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("m", [2, 3, 4, 5, 8])
|
||||
def test_scalar_multi_m_matches_fp32_reference(dist_init, m, dtype):
|
||||
"""The exact factored dequant across every M_COUNT instantiation
|
||||
(M=2,3 -> M_COUNT=2; M=4,5 -> M_COUNT=4 with a padded row; M=8 -> 8):
|
||||
the (y, z) correction and the nibble-slot activation sums must hold
|
||||
for every tile width, not just the M=1 fast path."""
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1247, dtype=dtype)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_single_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""Scalar z_count == 1 direct-store path vs the FP32 reference."""
|
||||
m = 1
|
||||
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1241, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_matches_fp32_reference(dist_init, dtype):
|
||||
"""WMMA split-K path (16x16_1w at the M=12 boundary) vs FP32 reference."""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1242, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_single_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""WMMA k_split == 1 direct-store path (K=256 -> compute_wmma_k_split
|
||||
returns 1 for every dispatch level) vs the FP32 reference."""
|
||||
m = 12 # smallest M on the WMMA side of the dispatch boundary
|
||||
outs, ref = _outputs_and_ref(M=m, K=256, N=512, seed=1243, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_large_m_matches_fp32_reference(dist_init, dtype):
|
||||
"""Large-M WMMA (128x64_k32, row-tiled scratch + reduce) vs FP32."""
|
||||
m = 128
|
||||
outs, ref = _outputs_and_ref(M=m, K=4096, N=512, seed=1244, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C. V7/V8 (128x64) no-split path: k_split == 1 -> direct store, no scratch,
|
||||
# no reduce pass (partials == nullptr).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The measured production-like shape from the W7900 benchmark campaign.
|
||||
# Routing arithmetic (see _assert_v7v8_no_split_routing): with (128, 64)
|
||||
# tiles, blocks_xy = ceil(25600/64) * ceil(512/128) = 400 * 4 = 1600 >=
|
||||
# 1500, so compute_wmma_k_split_mn returns 1. Smaller no-split shapes are
|
||||
# not practical: N would have to grow past ~24k at M=512 (or M past ~1k at
|
||||
# N=12k) to reach the 1500-block threshold, so this is effectively the
|
||||
# smallest production-realistic V7/V8 no-split shape.
|
||||
V7V8_NOSPLIT = (512, 25600, 6656) # (M, N, K)
|
||||
|
||||
|
||||
def _compute_wmma_k_split_mn(m, n, k, m_tile, n_tile):
|
||||
"""Faithful replica of compute_wmma_k_split_mn (C++ heuristic)."""
|
||||
blocks_xy = ((n + n_tile - 1) // n_tile) * ((m + m_tile - 1) // m_tile)
|
||||
if blocks_xy >= 1500:
|
||||
return 1
|
||||
if blocks_xy * 2 >= 1500 and k >= 512 and k % 32 == 0:
|
||||
return 2
|
||||
if blocks_xy * 4 >= 1500 and k >= 1024 and k % 64 == 0:
|
||||
return 4
|
||||
if k >= 1024 and k % 64 == 0:
|
||||
return 4
|
||||
if k >= 512 and k % 32 == 0:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
def _assert_v7v8_no_split_routing(dtype, m, n, k):
|
||||
"""Assert the shape deterministically routes to the V7/V8 128x64
|
||||
kernel with k_split == 1: WMMA dispatch (both dtypes M >= 12), the
|
||||
M >= 128 branch, and the no-split threshold."""
|
||||
assert m >= 12, "not the WMMA path"
|
||||
assert m >= 128, "below the V7/V8 (128x64) branch"
|
||||
assert _compute_wmma_k_split_mn(m, n, k, 128, 64) == 1, (
|
||||
"shape does not route to the k_split == 1 no-split path"
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_v7v8_no_split_bit_repeatable(dist_init, dtype):
|
||||
"""V7/V8 128x64_k32 with k_split == 1: single direct writer per output
|
||||
cell — no FP32 scratch, no reduce pass."""
|
||||
m, n, k = V7V8_NOSPLIT
|
||||
_assert_v7v8_no_split_routing(dtype, m, n, k)
|
||||
outs, _ = _outputs_and_ref(M=m, K=k, N=n, seed=1245, dtype=dtype, repeats=REPEATS)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_wmma_v7v8_no_split_matches_fp32_reference(dist_init, dtype):
|
||||
"""V7/V8 128x64_k32 no-split direct-store path vs the FP32 reference."""
|
||||
m, n, k = V7V8_NOSPLIT
|
||||
_assert_v7v8_no_split_routing(dtype, m, n, k)
|
||||
outs, ref = _outputs_and_ref(M=m, K=k, N=n, seed=1246, dtype=dtype)
|
||||
_assert_close_to_ref(outs[0], ref, _path_for(dtype, m), dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# D. Dispatch boundary: M=11 must stay scalar, M=12 must route to WMMA
|
||||
# (WMMA_MIN_M in csrc/rocm/q_gemm_rdna3.cu). Observed via the CUDA
|
||||
# profiler: the scalar and WMMA kernels have disjoint names.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _launched_kernel_names(fn) -> set:
|
||||
from torch.profiler import ProfilerActivity, profile
|
||||
|
||||
fn()
|
||||
torch.accelerator.synchronize()
|
||||
with profile(activities=[ProfilerActivity.CUDA]) as prof:
|
||||
fn()
|
||||
torch.accelerator.synchronize()
|
||||
return {
|
||||
e.name
|
||||
for e in prof.events()
|
||||
if "gemm_q4" in e.name or "reduce_partials" in e.name
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@gfx1100_only
|
||||
def test_dispatch_boundary(dist_init, dtype):
|
||||
"""The scalar/WMMA crossover sits between M=11 and M=12 for both
|
||||
dtypes; the boundary must be exact (see the WMMA_MIN_M table)."""
|
||||
|
||||
def run(m):
|
||||
outs, _ = _outputs_and_ref(M=m, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
return outs[0]
|
||||
|
||||
m11 = run(11)
|
||||
names11 = _launched_kernel_names(lambda: run(11))
|
||||
assert any("gemm_q4_kernel_rdna3" in n for n in names11), names11
|
||||
assert not any("wmma" in n for n in names11), names11
|
||||
|
||||
m12 = run(12)
|
||||
names12 = _launched_kernel_names(lambda: run(12))
|
||||
assert any("wmma" in n for n in names12), names12
|
||||
assert not any("gemm_q4_kernel_rdna3" in n for n in names12), names12
|
||||
|
||||
# Both sides of the boundary stay correct against the reference.
|
||||
_, ref11 = _outputs_and_ref(M=11, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
_, ref12 = _outputs_and_ref(M=12, K=4096, N=512, seed=1248, dtype=dtype)
|
||||
_assert_close_to_ref(m11, ref11, "scalar", dtype)
|
||||
_assert_close_to_ref(m12, ref12, "wmma", dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# E. In-kernel split-K reduction: stress the per-call ticket epilogue.
|
||||
# The fused reduction must stay deterministic and correct under
|
||||
# back-to-back unsynchronized calls, concurrent invocations on two
|
||||
# streams (per-call tickets must never share state), and CUDA-graph
|
||||
# capture/replay.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_splitk_100x_bit_repeatable(dist_init, dtype):
|
||||
"""100 back-to-back calls (no intermediate sync) on the split-K path."""
|
||||
outs, _ = _outputs_and_ref(M=1, K=4096, N=512, seed=1249, dtype=dtype, repeats=100)
|
||||
_assert_repeatable(outs)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_multistream_concurrent(dist_init, dtype):
|
||||
"""Two concurrent invocations on separate streams, distinct weights and
|
||||
activations: outputs must be bit-identical to the serial invocation of
|
||||
each (the arrival tickets are per-call, so cross-stream interference
|
||||
would show up here as a corrupted or unreduced output)."""
|
||||
layer_a, q_a, s_a = _build_layer(4096, 512, 4321, dtype)
|
||||
layer_b, q_b, s_b = _build_layer(4096, 512, 4322, dtype)
|
||||
wq_a, ws_a, wz_a = _prepare(layer_a, dtype, 4096, 512)
|
||||
wq_b, ws_b, wz_b = _prepare(layer_b, dtype, 4096, 512)
|
||||
torch.manual_seed(4323)
|
||||
x_a = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
x_b = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
|
||||
serial_a = _run_op(x_a, wq_a, wz_a, ws_a)
|
||||
serial_b = _run_op(x_b, wq_b, wz_b, ws_b)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
s1, s2 = torch.cuda.Stream(), torch.cuda.Stream()
|
||||
for _ in range(25):
|
||||
with torch.cuda.stream(s1):
|
||||
out_a = _run_op(x_a, wq_a, wz_a, ws_a)
|
||||
with torch.cuda.stream(s2):
|
||||
out_b = _run_op(x_b, wq_b, wz_b, ws_b)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(out_a, serial_a), "concurrent stream A corrupted"
|
||||
assert torch.equal(out_b, serial_b), "concurrent stream B corrupted"
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_graph_capture_replay(dist_init, dtype):
|
||||
"""The op must survive CUDA-graph capture: the replayed output must be
|
||||
bit-identical to the eager output and stable across repeated replays."""
|
||||
layer, q_int4, scales_gn = _build_layer(4096, 512, 4324, dtype)
|
||||
wq, ws, wz = _prepare(layer, dtype, 4096, 512)
|
||||
torch.manual_seed(4325)
|
||||
x = torch.randn(1, 4096, device=device, dtype=dtype)
|
||||
|
||||
eager = _run_op(x, wq, wz, ws)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
_run_op(x, wq, wz, ws) # warm-up side streams/allocator on capture path
|
||||
torch.accelerator.synchronize()
|
||||
with torch.cuda.graph(graph):
|
||||
captured = _run_op(x, wq, wz, ws)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(captured, eager), "graph replay differs from eager"
|
||||
graph.replay()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(captured, eager), "repeated replay not repeatable"
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_multi_row_tile_bit_repeatable(dist_init, dtype):
|
||||
"""Row tiling with ticket reuse: N % 16 == 8 bypasses the WMMA
|
||||
dispatch (N % 16 != 0), so M=100 runs scalar across two 64-row tiles
|
||||
(the second with gy > 1 y-blocks). Every tile's tickets must be armed,
|
||||
consumed, and re-armed correctly for all outputs to be written exactly
|
||||
once — and the result must be bit-repeatable."""
|
||||
outs, ref = _outputs_and_ref(
|
||||
M=100, K=4096, N=2056, seed=1250, dtype=dtype, repeats=20
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# F. Split-K geometry corners: clipped non-first z blocks, small split
|
||||
# counts, and non-128 group sizes.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("k,group", [(4224, 128), (2112, 32)])
|
||||
def test_scalar_clipped_tail_splitk(dist_init, dtype, k, group):
|
||||
"""K % 256 != 0 with z_count > 1: the LAST z block is clipped by end_k,
|
||||
its group prefetch guard (`k + 32 < end_k`) is what keeps the raw
|
||||
zero/scale loads in bounds, and the reduce tail loop runs (z=17 -> one
|
||||
remainder slice; z=9 -> one batch + one). K=2112/G=32 also puts two
|
||||
group switches inside the clipped tail block."""
|
||||
outs, ref = _outputs_and_ref(M=1, K=k, N=512, seed=1251, dtype=dtype, group=group)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("group", [32, 64])
|
||||
def test_scalar_small_group_matches_fp32_reference(dist_init, dtype, group):
|
||||
"""Group sizes below 128 make the group constants change every 32-K
|
||||
round (G=32) or three times per z block (G=64); the factored-dequant
|
||||
correction must track every change. G=32 is also the production MoE
|
||||
group size."""
|
||||
outs, ref = _outputs_and_ref(
|
||||
M=1, K=2048, N=512, seed=1252, dtype=dtype, group=group
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
_assert_close_to_ref(outs[0], ref, "scalar", dtype)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
def test_scalar_two_way_split_bit_repeatable(dist_init, dtype):
|
||||
"""z_count == 2 (K=512): the minimal split-K configuration — the old
|
||||
CAS bug's onset regime — and the pure-remainder reduce tail."""
|
||||
outs, _ = _outputs_and_ref(
|
||||
M=1, K=512, N=512, seed=1253, dtype=dtype, repeats=REPEATS
|
||||
)
|
||||
_assert_repeatable(outs)
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kernel-selection / gating tests for the ROCm RDNA3 W4A16 GPTQ kernel.
|
||||
|
||||
Verifies that ``choose_mp_linear_kernel`` resolves a supported W4A16 GPTQ
|
||||
config to ``RDNA3W4A16LinearKernel`` on gfx1100 (it is registered ahead of
|
||||
``TritonW4A16LinearKernel`` in the ROCm priority list), and that
|
||||
``RDNA3W4A16LinearKernel.can_implement`` rejects the configs it does not
|
||||
support so selection falls through to the next kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna3_w4a16_selection.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("RDNA3 W4A16 kernel is ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.kernels.linear import ( # noqa: E402
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( # noqa: E402
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.rdna3_w4a16 import ( # noqa: E402
|
||||
RDNA3W4A16LinearKernel,
|
||||
)
|
||||
from vllm.platforms.rocm import on_gfx1100 # noqa: E402
|
||||
from vllm.scalar_type import scalar_types # noqa: E402
|
||||
|
||||
WEIGHT_TYPE = scalar_types.uint4b8 # symmetric int4, bias = 8
|
||||
|
||||
# The kernel is only selectable when running on gfx1100 with the custom op
|
||||
# compiled in; otherwise can_implement rejects and selection falls through.
|
||||
gfx1100_only = pytest.mark.skipif(
|
||||
not (
|
||||
on_gfx1100()
|
||||
and hasattr(torch.ops, "_rocm_C")
|
||||
and hasattr(torch.ops._rocm_C, "gptq_gemm_rdna3")
|
||||
),
|
||||
reason="requires gfx1100 with the _rocm_C.gptq_gemm_rdna3 op built in",
|
||||
)
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_selection_prefers_rdna3(dtype):
|
||||
"""A supported W4A16 GPTQ config resolves to the RDNA3 kernel on gfx1100."""
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(1024, 256),
|
||||
partition_weight_shape=(1024, 256),
|
||||
weight_type=WEIGHT_TYPE,
|
||||
act_type=dtype,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
assert choose_mp_linear_kernel(config).__name__ == "RDNA3W4A16LinearKernel"
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize(
|
||||
"weight_type,group_size,N,full_k,expected_ok",
|
||||
[
|
||||
(scalar_types.uint4b8, 128, 256, 1024, True), # nominal: supported
|
||||
(scalar_types.uint4b8, -1, 256, 1024, False), # channelwise unsupported
|
||||
(scalar_types.uint4b8, 128, 252, 1024, False), # N not a multiple of 8
|
||||
(scalar_types.uint4b8, 96, 256, 1024, False), # group does not divide K
|
||||
(scalar_types.uint8b128, 128, 256, 1024, False), # wrong quant type
|
||||
],
|
||||
ids=["ok", "channelwise", "bad_n", "group_ndiv_k", "wrong_qtype"],
|
||||
)
|
||||
def test_can_implement(weight_type, group_size, N, full_k, expected_ok):
|
||||
"""can_implement gates on quant type, group size, and N divisibility."""
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(full_k, N),
|
||||
partition_weight_shape=(full_k, N),
|
||||
weight_type=weight_type,
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
)
|
||||
ok, reason = RDNA3W4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok, reason
|
||||
@@ -1,934 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ROCm Hybrid W4A16 kernel (HIP skinny + Triton prefill).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rdna_hybrid_w4a16.py`.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm only", allow_module_level=True)
|
||||
|
||||
pytest.importorskip("triton")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
hybrid_module = importlib.import_module(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.rdna_hybrid_w4a16"
|
||||
)
|
||||
RDNAHybridW4A16LinearKernel = hybrid_module.RDNAHybridW4A16LinearKernel
|
||||
pack_int4_exllama_shuffle = hybrid_module.pack_int4_exllama_shuffle
|
||||
SUPPORTED_GROUP_SIZES = hybrid_module.SUPPORTED_GROUP_SIZES
|
||||
MAX_SKINNY_BATCH_SIZE = hybrid_module.MAX_SKINNY_BATCH_SIZE
|
||||
LDS_CAPACITY_ELEMENTS = hybrid_module.LDS_CAPACITY_ELEMENTS
|
||||
MEDIUM_SKINNY_LIMIT_ELEMENTS = hybrid_module.MEDIUM_SKINNY_LIMIT_ELEMENTS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reference implementation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pack_zp_rows_for_kernel(zp_nkg: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack raw uint4 zero points along N: [N, G] int32 -> [N//8, G] int32.
|
||||
|
||||
Row n's nibble lands in word[n//8] at bits 4*(n%8).
|
||||
"""
|
||||
assert zp_nkg.dtype == torch.int32
|
||||
N, G = zp_nkg.shape
|
||||
assert N % 8 == 0
|
||||
shifts = (torch.arange(8, device=zp_nkg.device, dtype=torch.int32) * 4)[:, None]
|
||||
return torch.sum(
|
||||
(zp_nkg.view(N // 8, 8, G) & 0xF) << shifts, dim=1, dtype=torch.int32
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _rdna_hybrid_w4a16_reference(
|
||||
x_mk: torch.Tensor,
|
||||
w_int4_nk: torch.Tensor,
|
||||
scales_nkg: torch.Tensor,
|
||||
zp_nkg: torch.Tensor | None,
|
||||
group_size: int,
|
||||
bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference for the Hybrid W4A16 op.
|
||||
|
||||
x_mk: [M, K] fp16/bf16
|
||||
w_int4_nk: [N, K] int32 with raw uint4 values in [0, 15]
|
||||
scales_nkg: [N, K//G] fp16/bf16
|
||||
zp_nkg: [N, K//G] int32 raw zero points in [0, 15], or None for
|
||||
symmetric (uint4b8, dequant subtracts 8)
|
||||
"""
|
||||
G = group_size
|
||||
N, K = w_int4_nk.shape
|
||||
assert K % G == 0
|
||||
s_full = scales_nkg.repeat_interleave(G, dim=1).to(torch.float32) # [N, K]
|
||||
if zp_nkg is None:
|
||||
z_full = torch.full((N, K), 8.0, device=x_mk.device, dtype=torch.float32)
|
||||
else:
|
||||
z_full = zp_nkg.repeat_interleave(G, dim=1).to(torch.float32)
|
||||
w_fp = (w_int4_nk.to(torch.float32) - z_full) * s_full # [N, K]
|
||||
out = x_mk.to(torch.float32) @ w_fp.t() # [M, N]
|
||||
if bias is not None:
|
||||
out = out + bias.to(torch.float32)
|
||||
return out.to(x_mk.dtype)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Forward correctness: decode (HIP skinny) + prefill (Triton)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"M",
|
||||
[1, MAX_SKINNY_BATCH_SIZE, MAX_SKINNY_BATCH_SIZE + 1, 64],
|
||||
ids=["M=1_decode", "M=5_decode", "M=6_prefill", "M=64_prefill"],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M):
|
||||
"""Smoke test the registered custom op for both decode and prefill batches.
|
||||
|
||||
Verifies the dispatch logic in `_rdna_hybrid_w4a16_apply_impl`:
|
||||
- supported M and K*M: HIP wvSplitK_int4_g
|
||||
- otherwise: Triton prefill kernel
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
K, N = 1024, 256
|
||||
assert K % group_size == 0 and K % 8 == 0 and N % 8 == 0
|
||||
|
||||
# Activations.
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
|
||||
# Weights as raw uint4 in [N, K], packed to ExLlama shuffle [N, K//8].
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q_i32 = pack_int4_exllama_shuffle(w_int4_nk).contiguous() # [N, K//8] int32
|
||||
w_q = w_q_i32.view(torch.int8) # same bytes viewed as int8 [N, K//2]
|
||||
|
||||
# Scales [N, K//G] in act dtype.
|
||||
scales_nkg = (
|
||||
0.05 * torch.rand((N, K // group_size), device=device, dtype=torch.float32)
|
||||
).to(dtype)
|
||||
|
||||
# Optional raw zero points, [N, K//G] int32 for the reference and packed
|
||||
# along N for the op.
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(
|
||||
0, 16, (N, K // group_size), device=device, dtype=torch.int32
|
||||
)
|
||||
w_zp = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
else:
|
||||
zp_nkg = None
|
||||
w_zp = None
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
x_mk,
|
||||
w_q,
|
||||
scales_nkg,
|
||||
w_zp,
|
||||
None, # bias
|
||||
num_compute_units(),
|
||||
group_size,
|
||||
)
|
||||
|
||||
ref = _rdna_hybrid_w4a16_reference(
|
||||
x_mk, w_int4_nk, scales_nkg, zp_nkg, group_size, bias=None
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
|
||||
def test_rdna_hybrid_w4a16_apply_with_bias(dtype, M):
|
||||
"""Bias is added correctly on both decode and prefill paths."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
set_random_seed(0)
|
||||
K, N, G = 1024, 128, 128
|
||||
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q_i32 = pack_int4_exllama_shuffle(w_int4_nk).contiguous()
|
||||
w_q = w_q_i32.view(torch.int8)
|
||||
scales_nkg = (
|
||||
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)
|
||||
).to(dtype)
|
||||
bias = torch.randn(N, device=device, dtype=dtype) * 0.1
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
x_mk,
|
||||
w_q,
|
||||
scales_nkg,
|
||||
None,
|
||||
bias,
|
||||
num_compute_units(),
|
||||
G,
|
||||
)
|
||||
ref = _rdna_hybrid_w4a16_reference(x_mk, w_int4_nk, scales_nkg, None, G, bias=bias)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# pack_int4_exllama_shuffle round-trips correctly
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_pack_int4_exllama_shuffle_layout():
|
||||
"""Pack 8 K-values per int32 in interleave [0,2,4,6,1,3,5,7] order."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
set_random_seed(0)
|
||||
N, K = 4, 16
|
||||
w = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
packed = pack_int4_exllama_shuffle(w)
|
||||
assert packed.shape == (N, K // 8) and packed.dtype == torch.int32
|
||||
|
||||
# Manual unshuffle using ExLlama shifts [0,16,4,20,8,24,12,28].
|
||||
shifts = torch.tensor(
|
||||
[0, 16, 4, 20, 8, 24, 12, 28], device=device, dtype=torch.int32
|
||||
)
|
||||
unshuffled = (packed.unsqueeze(-1) >> shifts) & 0xF
|
||||
unshuffled = unshuffled.reshape(N, K)
|
||||
torch.testing.assert_close(unshuffled, w)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# process_weights_after_loading: layout repack and zp normalization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8]."""
|
||||
assert w_int4_kn.dtype == torch.int32
|
||||
K, N = w_int4_kn.shape
|
||||
assert K % 8 == 0
|
||||
out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device)
|
||||
for i in range(8):
|
||||
out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4)
|
||||
return out.contiguous()
|
||||
|
||||
|
||||
def _pack_int4_along_n_for_zp(zp_int4_gn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 zero points along N: [G, N] -> [G, N//8] int32 (CT layout)."""
|
||||
assert zp_int4_gn.dtype == torch.int32
|
||||
G, N = zp_int4_gn.shape
|
||||
assert N % 8 == 0
|
||||
shifts = torch.arange(8, device=zp_int4_gn.device, dtype=torch.int32) * 4
|
||||
return torch.sum(
|
||||
(zp_int4_gn.view(G, N // 8, 8) & 0xF) << shifts, dim=2, dtype=torch.int32
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _build_dummy_layer(
|
||||
w_ckpt_nk8: torch.Tensor,
|
||||
scales_ckpt_nkg: torch.Tensor,
|
||||
zeros_ckpt: torch.Tensor | None,
|
||||
):
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"weight_packed",
|
||||
PackedvLLMParameter(
|
||||
data=w_ckpt_nk8,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=1,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_ckpt_nkg,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
),
|
||||
)
|
||||
if zeros_ckpt is not None:
|
||||
layer.register_parameter(
|
||||
"weight_zero_point",
|
||||
PackedColumnParameter(
|
||||
data=zeros_ckpt,
|
||||
weight_loader=weight_loader,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=0,
|
||||
),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
|
||||
def test_rdna_hybrid_w4a16_process_weights_symmetric_repack(group_size, dist_init):
|
||||
"""uint4b8 (symmetric): w_q -> [N, K//8] int8 ExLlama shuffle, no zp param."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
K, N = 256, 128
|
||||
G = group_size
|
||||
assert K % G == 0
|
||||
|
||||
# Reference unpacked weights, then pack into CT checkpoint layout [N, K//8].
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn)
|
||||
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
layer = _build_dummy_layer(w_ckpt_nk8, scales_ckpt_nkg, zeros_ckpt=None)
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# Skinny weight is stored once as int8 [N, K//2]; the Triton path
|
||||
# reinterprets it as int32 [N, K//8] via a view (no separate parameter).
|
||||
assert layer.weight_packed.dtype == torch.int8
|
||||
assert tuple(layer.weight_packed.shape) == (N, K // 2)
|
||||
w_q_i32 = layer.weight_packed.view(torch.int32)
|
||||
assert tuple(w_q_i32.shape) == (N, K // 8)
|
||||
|
||||
expected_packed = pack_int4_exllama_shuffle(w_int4_kn.t().contiguous())
|
||||
torch.testing.assert_close(w_q_i32, expected_packed)
|
||||
|
||||
# Scales: [N, K//G] (skinny layout, no transpose since CT already had it).
|
||||
assert tuple(layer.weight_scale.shape) == (N, K // G)
|
||||
torch.testing.assert_close(layer.weight_scale, scales_ckpt_nkg)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group_size", SUPPORTED_GROUP_SIZES)
|
||||
def test_rdna_hybrid_w4a16_process_weights_asymmetric_repack(group_size, dist_init):
|
||||
"""uint4 (asymmetric): zero points are packed [N//8, K//G] int32."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
K, N = 256, 128
|
||||
G = group_size
|
||||
assert K % G == 0 and N % 8 == 0
|
||||
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn)
|
||||
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
# CT zero-point layout is N-packed: [N//8, K//G] int32.
|
||||
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
zeros_packed_gn8 = _pack_int4_along_n_for_zp(zeros_int4_gn) # [K//G, N//8]
|
||||
zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G]
|
||||
|
||||
layer = _build_dummy_layer(w_ckpt_nk8, scales_ckpt_nkg, zeros_ckpt=zeros_ckpt_n8kg)
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
)
|
||||
kernel = RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# Zero-points: [N//8, K//G] int32, row n's nibble at word[n//8]
|
||||
# bits 4*(n%8).
|
||||
assert layer.weight_zero_point.dtype == torch.int32
|
||||
assert tuple(layer.weight_zero_point.shape) == (N // 8, K // G)
|
||||
expected_zp = _pack_zp_rows_for_kernel(zeros_int4_gn.t().contiguous())
|
||||
torch.testing.assert_close(layer.weight_zero_point, expected_zp)
|
||||
|
||||
# Quantized weights match symmetric path's layout regardless of zp.
|
||||
w_q_i32 = layer.weight_packed.view(torch.int32)
|
||||
assert tuple(w_q_i32.shape) == (N, K // 8)
|
||||
expected_packed = pack_int4_exllama_shuffle(w_int4_kn.t().contiguous())
|
||||
torch.testing.assert_close(w_q_i32, expected_packed)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# can_implement enforces the supported-group-size policy
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok", [(32, True), (64, True), (128, True), (256, False)]
|
||||
)
|
||||
def test_hybrid_can_implement_group_size(group_size, expected_ok):
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
K, N = 1024, 256
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=group_size,
|
||||
zero_points=False,
|
||||
)
|
||||
ok, _ = RDNAHybridW4A16LinearKernel.can_implement(config)
|
||||
assert ok is expected_ok
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for the HIP wvSplitK_int4_g decode kernel
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _hip_skinny_reference(
|
||||
a_mk: torch.Tensor,
|
||||
w_int4_nk: torch.Tensor,
|
||||
scales_nkg: torch.Tensor,
|
||||
*,
|
||||
group_size: int,
|
||||
zp_nkg: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference for HIP skinny: C = A @ (W - zp) * S.
|
||||
|
||||
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
|
||||
None for symmetric uint4b8, where the kernel subtracts a constant 8.
|
||||
"""
|
||||
K = a_mk.shape[1]
|
||||
N = w_int4_nk.shape[0]
|
||||
num_groups = K // group_size
|
||||
|
||||
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
|
||||
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
|
||||
s = scales_nkg.to(torch.float32).unsqueeze(-1)
|
||||
w_dequant = ((w_g - zp) * s).view(N, K)
|
||||
|
||||
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G",
|
||||
[
|
||||
(1, 256, 256, 32),
|
||||
(1, 256, 256, 64),
|
||||
(1, 512, 256, 128),
|
||||
(2, 512, 256, 64),
|
||||
(3, 256, 512, 64),
|
||||
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
|
||||
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
|
||||
(4, 512, 256, 32),
|
||||
(4, 256, 512, 64),
|
||||
(5, 512, 256, 128),
|
||||
(5, 256, 256, 32),
|
||||
# N == 8 is exactly one packed zero-point word.
|
||||
(1, 256, 8, 32),
|
||||
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
|
||||
# unrolled block; every K above is below that bound, so without these
|
||||
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
|
||||
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
|
||||
(1, 4096, 256, 128),
|
||||
(2, 2560, 256, 32),
|
||||
(3, 4096, 256, 64),
|
||||
(4, 2560, 256, 128),
|
||||
(5, 4096, 256, 32),
|
||||
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
|
||||
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
|
||||
# tight case there: each lane's A_CHUNK spans exactly one scale group.
|
||||
(1, 4096, 256, 32),
|
||||
(1, 8192, 256, 128),
|
||||
(1, 16384, 256, 64),
|
||||
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
|
||||
# walks K in chunks; elsewhere the op rejects these, as the layer
|
||||
# routes them to Triton. The per-row window shrinks as the batch grows,
|
||||
# so these cover a single reload and several, with and without a short
|
||||
# final chunk. 20992 is not a multiple of the K step either, so it
|
||||
# crosses a chunk boundary *and* has a ragged K tail.
|
||||
(2, 20480, 512, 64),
|
||||
(3, 16384, 512, 32),
|
||||
(4, 16384, 512, 128),
|
||||
(5, 21504, 512, 32),
|
||||
(5, 20992, 512, 128),
|
||||
],
|
||||
)
|
||||
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
|
||||
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
|
||||
pytest.skip("deep-K HIP path is gfx115x only")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
|
||||
b_packed_i32 = pack_int4_exllama_shuffle(w_int4_nk)
|
||||
b_packed_i8 = b_packed_i32.view(torch.int8)
|
||||
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
|
||||
zp_nkg = None
|
||||
zp_packed = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("G", [32, 128])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
|
||||
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
|
||||
"""Scale and zero-point rows may be padded past K // group_size.
|
||||
|
||||
Both paths index them by their own row stride, so a padded row is read
|
||||
correctly rather than bleeding into the next row. M picks the path: the
|
||||
HIP skinny kernel at 2, the Triton prefill kernel at 64.
|
||||
"""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
K, N = 2048, 256
|
||||
num_groups = K // G
|
||||
pad = 7
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
|
||||
|
||||
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
|
||||
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
|
||||
scales_padded[:, :num_groups] = scales
|
||||
scales_view = scales_padded[:, :num_groups]
|
||||
assert scales_view.stride(0) == num_groups + pad
|
||||
|
||||
zp_nkg = zp_view = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
zp_padded = torch.zeros(
|
||||
(N // 8, num_groups + pad), device=device, dtype=torch.int32
|
||||
)
|
||||
zp_padded[:, :num_groups] = zp_packed
|
||||
zp_view = zp_padded[:, :num_groups]
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
|
||||
)
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for the full hybrid dispatch (HIP decode + Triton prefill)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G",
|
||||
[
|
||||
(1, 256, 256, 64),
|
||||
(1, 512, 256, 32),
|
||||
(1, 512, 256, 128),
|
||||
# Past LDS: HIP on gfx115x, Triton elsewhere.
|
||||
(2, 20480, 256, 64),
|
||||
(32, 512, 256, 64),
|
||||
(64, 1024, 256, 128),
|
||||
],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
|
||||
"""Test the full hybrid dispatch via the custom op."""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
|
||||
b_packed_i32 = pack_int4_exllama_shuffle(w_int4_nk)
|
||||
b_packed_i8 = b_packed_i32.view(torch.int8)
|
||||
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, b_packed_i8, scales, None, None, cu_count, G
|
||||
)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops, "_rocm_C")
|
||||
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
|
||||
reason="wvSplitK_int4_g not built",
|
||||
)
|
||||
def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
|
||||
"""Act-dtype zero points raise instead of being read as packed words.
|
||||
|
||||
Both layouts are 2D with compatible extents, so only the dtype separates
|
||||
them; misreading one as the other returns wrong numbers silently.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
K, N, G, M = 256, 64, 128, 1
|
||||
a = torch.randn((M, K), device=device, dtype=torch.float16)
|
||||
w = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
|
||||
torch.int8
|
||||
)
|
||||
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
zp_unpacked = torch.zeros((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
|
||||
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops, "_rocm_C")
|
||||
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
|
||||
reason="wvSplitK_int4_g not built",
|
||||
)
|
||||
def test_wvsplitk_int4_g_rejects_undersized_weight_view():
|
||||
"""A sufficient row stride does not prove the row holds K/2 packed bytes.
|
||||
|
||||
Row padding makes stride(0) > K/2 legitimate, but the logical size still
|
||||
has to be checked: this view has a valid stride and half the required
|
||||
bytes, and without the size check the kernel reads past it.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
K, N, G, M = 512, 256, 128, 1
|
||||
a = torch.randn((M, K), device=device, dtype=torch.float16)
|
||||
# Full-size backing allocation, so a missing check misreads rather than
|
||||
# running off the end of the buffer.
|
||||
backing = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
|
||||
torch.int8
|
||||
)
|
||||
undersized = backing[:, : K // 4]
|
||||
assert undersized.stride(0) == K // 2
|
||||
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"must contain M\*K/2 bytes"):
|
||||
ops.wvSplitK_int4_g(undersized, a, scales, num_compute_units(), G, None, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gfx11 weight row-stride padding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_weight_pad_bytes = hybrid_module._weight_pad_bytes
|
||||
_act_pad_bytes = hybrid_module._act_pad_bytes
|
||||
_WEIGHT_CLIFF_BYTES = hybrid_module._WEIGHT_CLIFF_BYTES
|
||||
_ACT_CLIFF_BYTES = hybrid_module._ACT_CLIFF_BYTES
|
||||
_STRIDE_PAD_BYTES = hybrid_module._STRIDE_PAD_BYTES
|
||||
|
||||
|
||||
def test_weight_pad_bytes_only_moves_strides_on_the_cliff():
|
||||
"""Pad 1024 B multiples, leave everything else dense."""
|
||||
# On the cliff (K % 2048 == 0): 1024 B multiples.
|
||||
for row_bytes in (1024, 2048, 4096, 5120, 6144, 7168, 8192):
|
||||
assert _weight_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
|
||||
|
||||
# Off it, including strides that are 512 B multiples but not 1024 B ones:
|
||||
# padding those measured as a real loss on gfx1151 (-14% at M=1 on a
|
||||
# 4864 B row).
|
||||
for row_bytes in (1280, 1536, 2560, 4864, 9472, 12800):
|
||||
assert _weight_pad_bytes(row_bytes) == 0
|
||||
|
||||
# The padded stride is never back on the cliff.
|
||||
for row_bytes in range(16, 16384, 16):
|
||||
padded = row_bytes + _weight_pad_bytes(row_bytes)
|
||||
assert padded % _WEIGHT_CLIFF_BYTES != 0 or row_bytes % _WEIGHT_CLIFF_BYTES
|
||||
|
||||
|
||||
def test_act_pad_bytes_only_moves_strides_on_the_cliff():
|
||||
"""Activations sit on a wider cliff than the packed weight: 2048 B."""
|
||||
# On the cliff (K % 1024 == 0 for a 2-byte dtype): 2048 B multiples.
|
||||
for row_bytes in (2048, 4096, 8192, 16384, 20480, 24576, 43008):
|
||||
assert _act_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
|
||||
|
||||
# Off it, including 1024 B multiples that are not 2048 B ones -- the
|
||||
# weight cliff must not be applied to activations.
|
||||
for row_bytes in (1024, 3072, 5120, 10752, 19456, 37888):
|
||||
assert _act_pad_bytes(row_bytes) == 0
|
||||
|
||||
# The padded stride is never back on the cliff.
|
||||
for row_bytes in range(16, 65536, 16):
|
||||
padded = row_bytes + _act_pad_bytes(row_bytes)
|
||||
assert padded % _ACT_CLIFF_BYTES != 0 or row_bytes % _ACT_CLIFF_BYTES
|
||||
|
||||
|
||||
def _row_padded_copy(t: torch.Tensor, pad_cols: int) -> torch.Tensor:
|
||||
"""Copy of ``t`` whose stride(0) is ``t.shape[1] + pad_cols``."""
|
||||
rows, cols = t.shape
|
||||
buf = torch.empty((rows, cols + pad_cols), dtype=t.dtype, device=t.device)
|
||||
buf[:, :cols].copy_(t)
|
||||
return buf[:, :cols]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
|
||||
def test_weight_row_padding_does_not_change_results(dtype, has_zp, M):
|
||||
"""Padding the weight rows is a pure layout change.
|
||||
|
||||
Both the HIP skinny kernel (M small) and the Triton prefill kernel take the
|
||||
weight row stride from the tensor, so a padded layout must produce exactly
|
||||
the same numbers as the dense one.
|
||||
"""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
K, N, G = 512, 256, 128
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
zp = (
|
||||
_pack_zp_rows_for_kernel(
|
||||
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
)
|
||||
if has_zp
|
||||
else None
|
||||
)
|
||||
|
||||
# 32 int32 columns = 128 B of pad, the production pad size.
|
||||
w_q_pad = _row_padded_copy(w_q.view(torch.int32), 32).view(torch.int8)
|
||||
assert w_q_pad.stride(0) == w_q.stride(0) + _STRIDE_PAD_BYTES
|
||||
|
||||
cu_count = num_compute_units()
|
||||
dense = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, w_q, scales, zp, None, cu_count, G
|
||||
)
|
||||
padded = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, w_q_pad, scales, zp, None, cu_count, G
|
||||
)
|
||||
torch.testing.assert_close(padded, dense, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hybrid_module._on_gfx1151(),
|
||||
reason="row-stride padding is only enabled on gfx1151",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"K,expected_weight_stride",
|
||||
[
|
||||
(512, 256), # 256 B row: off the cliff, stored dense
|
||||
(8192, 4096 + 128), # 4096 B row: on the cliff, padded
|
||||
],
|
||||
)
|
||||
def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
|
||||
"""The stored weight row stride follows the padding rule."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
set_random_seed(0)
|
||||
N, G = 128, 128
|
||||
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
layer = _build_dummy_layer(
|
||||
_pack_int4_along_k_to_ckpt(w_int4_kn),
|
||||
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16),
|
||||
zeros_ckpt=None,
|
||||
)
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
)
|
||||
RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
).process_weights_after_loading(layer)
|
||||
|
||||
# Weight rows: stride is in int8 elements, i.e. bytes.
|
||||
assert layer.weight_packed.stride(0) == expected_weight_stride
|
||||
# The int32 view the Triton path uses survives the padded stride.
|
||||
assert tuple(layer.weight_packed.view(torch.int32).shape) == (N, K // 8)
|
||||
# Metadata is untouched by this change.
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,gfx115x,expected_path",
|
||||
[
|
||||
(4, 8192, False, "hip"),
|
||||
(4, 9728, False, "hip"),
|
||||
(4, 9856, False, "triton"),
|
||||
(4, 9856, True, "hip"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
|
||||
],
|
||||
ids=[
|
||||
"regular_limit",
|
||||
"medium_range",
|
||||
"above_medium",
|
||||
"above_medium_gfx115x",
|
||||
"batch_too_large",
|
||||
],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(
|
||||
M, K, gfx115x, expected_path, monkeypatch
|
||||
):
|
||||
"""Route only supported regular and medium skinny shapes to HIP.
|
||||
|
||||
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
|
||||
HIP there; the batch bound holds on every target.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
N, G = 16, 128
|
||||
x_mk = torch.empty((M, K), dtype=torch.float16)
|
||||
w_q = torch.empty((N, K // 2), dtype=torch.int8)
|
||||
scales = torch.empty((N, K // G), dtype=torch.float16)
|
||||
called = []
|
||||
|
||||
def fake_hip(*args, **kwargs):
|
||||
called.append("hip")
|
||||
return torch.empty((M, N), dtype=x_mk.dtype)
|
||||
|
||||
def fake_triton(*args, **kwargs):
|
||||
called.append("triton")
|
||||
return torch.empty((M, N), dtype=x_mk.dtype)
|
||||
|
||||
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
|
||||
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
|
||||
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
|
||||
|
||||
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
|
||||
x_mk, w_q, scales, None, None, 40, G
|
||||
)
|
||||
|
||||
assert output.shape == (M, N)
|
||||
assert called == [expected_path]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
def test_rdna_hybrid_w4a16_medium_skinny_matches_reference(dtype, has_zp):
|
||||
"""Validate the HIP medium path against an FP32 dequantization reference."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
M, K, N, G = 4, 9728, 256, 128
|
||||
assert LDS_CAPACITY_ELEMENTS < K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS
|
||||
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q = pack_int4_exllama_shuffle(w_int4_nk).contiguous().view(torch.int8)
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
zp = (
|
||||
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
if has_zp
|
||||
else None
|
||||
)
|
||||
w_zp = _pack_zp_rows_for_kernel(zp) if zp is not None else None
|
||||
|
||||
output = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
x_mk, w_q, scales, w_zp, None, num_compute_units(), G
|
||||
)
|
||||
reference = _rdna_hybrid_w4a16_reference(x_mk, w_int4_nk, scales, zp, G, bias=None)
|
||||
|
||||
torch.testing.assert_close(output, reference, rtol=2e-2, atol=2e-2)
|
||||
@@ -1,137 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# This is a test for the AITER group_fp8_quant op.
|
||||
# It tests if the AITER op is
|
||||
# 1. correctly defined the relationship between
|
||||
# implementation and fake function
|
||||
# 2. can be used with torch.compile
|
||||
# 3. can be used with CUDA graphs
|
||||
# This file will be skipped if AITER is not installed
|
||||
# and the platform is not ROCm.
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# this import statement is needed to ensure the ops are registered
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Check if aiter package is installed
|
||||
aiter_available = importlib.util.find_spec("aiter") is not None
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and aiter_available),
|
||||
reason="AITER ops are only available on ROCm with aiter package installed",
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_group_fp8_quant_fake_implementation():
|
||||
"""Test that the fake implementation is correctly
|
||||
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant."""
|
||||
# Create test tensors
|
||||
M = 128
|
||||
N = 4096
|
||||
group_size = 128
|
||||
|
||||
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
# Verify the op's fake implementation using torch.library.opcheck
|
||||
# This checks that the fake function returns tensors with correct shapes and dtypes
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_group_fp8_quant,
|
||||
(input_tensor, group_size),
|
||||
test_utils=("test_faketensor",),
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_aiter_group_fp8_quant_torch_compile_with_cudagraph():
|
||||
"""Test that rocm_aiter_ops.group_fp8_quant
|
||||
with group size 128 can be used with
|
||||
torch.compile in cudagraph mode."""
|
||||
# Create test tensors
|
||||
M = 128
|
||||
N = 4096
|
||||
group_size = 128
|
||||
|
||||
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
# Define a function that uses the op
|
||||
def group_fp8_quant_fn(x):
|
||||
return rocm_aiter_ops.group_fp8_quant(x, group_size)
|
||||
|
||||
# Compile with cudagraph mode
|
||||
compiled_fn = torch.compile(
|
||||
group_fp8_quant_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
# Run eager mode
|
||||
x_fp8_eager, scales_eager = group_fp8_quant_fn(input_tensor)
|
||||
|
||||
# Run compiled version (first run will trigger compilation)
|
||||
x_fp8_compiled, scales_compiled = compiled_fn(input_tensor)
|
||||
|
||||
# Verify shapes match
|
||||
assert x_fp8_compiled.shape == x_fp8_eager.shape
|
||||
assert scales_compiled.shape == scales_eager.shape
|
||||
|
||||
# Verify expected shapes
|
||||
assert x_fp8_compiled.shape == (M, N)
|
||||
expected_scale_cols = (N + group_size - 1) // group_size
|
||||
assert scales_compiled.shape == (M, expected_scale_cols)
|
||||
|
||||
# Verify results match
|
||||
assert torch.allclose(
|
||||
x_fp8_compiled.to(torch.float32),
|
||||
x_fp8_eager.to(torch.float32),
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
assert torch.allclose(scales_compiled, scales_eager, rtol=1e-3, atol=1e-3)
|
||||
|
||||
# Test with different input (reusing compiled graph)
|
||||
input_tensor_2 = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
|
||||
x_fp8_eager_2, scales_eager_2 = group_fp8_quant_fn(input_tensor_2)
|
||||
x_fp8_compiled_2, scales_compiled_2 = compiled_fn(input_tensor_2)
|
||||
|
||||
# Verify second run also produces correct results
|
||||
assert torch.allclose(
|
||||
x_fp8_compiled_2.to(torch.float32),
|
||||
x_fp8_eager_2.to(torch.float32),
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
assert torch.allclose(scales_compiled_2, scales_eager_2, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def test_rocm_aiter_group_fp8_quant_different_shapes():
|
||||
"""Test rocm_aiter_ops.group_fp8_quant with different input shapes."""
|
||||
group_size = 128
|
||||
|
||||
test_shapes = [
|
||||
(64, 2048),
|
||||
(256, 8192),
|
||||
(32, 1024),
|
||||
(512, 4096),
|
||||
]
|
||||
|
||||
for M, N in test_shapes:
|
||||
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
x_fp8, scales = rocm_aiter_ops.group_fp8_quant(input_tensor, group_size)
|
||||
|
||||
# Verify shapes
|
||||
assert x_fp8.shape == (M, N)
|
||||
expected_scale_cols = (N + group_size - 1) // group_size
|
||||
assert scales.shape == (M, expected_scale_cols)
|
||||
|
||||
# Verify dtypes
|
||||
from aiter import dtypes
|
||||
|
||||
assert x_fp8.dtype == dtypes.fp8
|
||||
assert scales.dtype == torch.float32
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end smoke test for CT W4A16 models on ROCm.
|
||||
|
||||
This validates that a real compressed-tensors W4A16 model can run inference
|
||||
end-to-end (which will exercise the Triton W4A16 kernel when selected).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_rocm_compressed_tensors_w4a16.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_path",
|
||||
[
|
||||
"nm-testing/tinyllama-oneshot-w4a16-group128-v2",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm")
|
||||
def test_rocm_compressed_tensors_w4a16_e2e(
|
||||
vllm_runner, example_prompts, model_path, max_tokens
|
||||
):
|
||||
# Use fp16 activations for maximum compatibility.
|
||||
# gpu_memory_utilization lowered to work on shared nodes.
|
||||
with vllm_runner(
|
||||
model_path, dtype="float16", gpu_memory_utilization=0.3
|
||||
) as vllm_model:
|
||||
# If the W4A16 kernel is broken, this will typically throw.
|
||||
vllm_model.generate_greedy(example_prompts, max_tokens=max_tokens)
|
||||
@@ -1,729 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm AITER FP4 and MXFP4 tests.
|
||||
|
||||
This file keeps the ROCm-specific FP4 coverage:
|
||||
- ROCm AITER FP4 env and enablement gates
|
||||
- large-shape MXFP4 wrapper checks through vLLM's public helper op
|
||||
- AITER Triton MXFP4 quant format and determinism
|
||||
- FP4 GEMM, preshuffled-scale, hardware-quant, and skinny decode
|
||||
paths
|
||||
|
||||
Generic MXFP4 wrapper/reference checks already live in
|
||||
``tests/kernels/quantization/test_quark.py``.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import _assert_accurate, _assert_deterministic
|
||||
from tests.quantization.reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
LLAMA_SHAPES = [
|
||||
(64, 4096),
|
||||
(64, 11008),
|
||||
(32, 14336),
|
||||
]
|
||||
GEMM_ATOL = 0.5
|
||||
GEMM_RTOL = 0.0
|
||||
GEMM_MAX_VIOLATION_FACTOR = 2.0
|
||||
A4W4_GEMM_PASS_RATES = {
|
||||
(64, 128, 64): 0.9999,
|
||||
(128, 256, 128): 0.9999,
|
||||
(128, 4096, 4096): 0.9999,
|
||||
(256, 4096, 11008): 0.9999,
|
||||
(64, 8192, 28672): 0.99765,
|
||||
}
|
||||
SKINNY_GEMM_PASS_RATES = {
|
||||
(4096, 4096): 0.9999,
|
||||
(4096, 11008): 0.99235,
|
||||
(8192, 8192): 0.99765,
|
||||
}
|
||||
PRESHUFFLED_SHAPES = [
|
||||
(64, 4096, 8192),
|
||||
(32, 16384, 16384),
|
||||
]
|
||||
|
||||
|
||||
def _reload_envs():
|
||||
import vllm.envs as envs
|
||||
|
||||
return importlib.reload(envs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_rocm_env_state():
|
||||
"""Restore global env + AITER flag state after every test."""
|
||||
yield
|
||||
_reload_envs()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _assert_aiter_supported() -> None:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
assert is_aiter_found_and_supported(), (
|
||||
"aiter is required on supported ROCm hardware for this test"
|
||||
)
|
||||
|
||||
|
||||
def _format_observed_rate(count: int, total: int) -> str:
|
||||
return f"{count / total:.4%} ({count}/{total})"
|
||||
|
||||
|
||||
def _format_allowed_rate(rate: float, total: int) -> str:
|
||||
allowed_count = int(rate * total)
|
||||
return f"{rate:.4%} (<= {allowed_count}/{total})"
|
||||
|
||||
|
||||
def _print_close_stats(
|
||||
label: str,
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
atol: float,
|
||||
rtol: float,
|
||||
pass_rate: float = 1.0,
|
||||
max_atol: float | None = None,
|
||||
) -> None:
|
||||
abs_diff = (actual - expected).abs().float().flatten()
|
||||
expected_abs = expected.abs().float().flatten()
|
||||
allowed = atol + rtol * expected_abs
|
||||
within = abs_diff <= allowed
|
||||
|
||||
total = abs_diff.numel()
|
||||
passed = int(within.sum().item())
|
||||
failed = total - passed
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
|
||||
max_abs = abs_diff.max().item()
|
||||
mean_abs = abs_diff.mean().item()
|
||||
p99_abs = torch.quantile(abs_diff, 0.99).item()
|
||||
p999_abs = torch.quantile(abs_diff, 0.999).item()
|
||||
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
|
||||
|
||||
msg = (
|
||||
"[rocm_aiter_fp4] "
|
||||
f"{label}: "
|
||||
f"pass={passed / total:.4%} ({passed}/{total}) "
|
||||
f"fail={_format_observed_rate(failed, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"atol={atol:g} "
|
||||
f"rtol={rtol:g} "
|
||||
)
|
||||
if max_atol is not None:
|
||||
above_max_count = int((abs_diff > max_atol).sum().item())
|
||||
msg += (
|
||||
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
|
||||
)
|
||||
msg += (
|
||||
f"max_abs={max_abs:.6g} "
|
||||
f"mean_abs={mean_abs:.6g} "
|
||||
f"p99_abs={p99_abs:.6g} "
|
||||
f"p999_abs={p999_abs:.6g} "
|
||||
f"worst_ratio={worst_ratio:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if failed > 0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
|
||||
# Env and enablement tests ------------------------------------------------
|
||||
|
||||
|
||||
def test_fp4_env_defaults():
|
||||
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("use_aiter", "use_fp4bmm"),
|
||||
[(True, True), (True, False), (False, True)],
|
||||
)
|
||||
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
use_aiter,
|
||||
use_fp4bmm,
|
||||
monkeypatch,
|
||||
):
|
||||
"""The ROCm FP4 AITER gates should depend only on the env toggles and the
|
||||
gfx950 hardware check.
|
||||
|
||||
Expected values are derived from the *actual* platform rather than a patched
|
||||
arch (gfx950 -> MI355, gfx942 -> MI300). A failure in this group therefore
|
||||
points straight at the hardware it ran on instead of a mocked value.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
on_gfx950_value = on_gfx950()
|
||||
expected_asm_gemm = use_aiter and on_gfx950_value
|
||||
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
|
||||
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
|
||||
|
||||
|
||||
# Large-shape MXFP4 wrapper tests -----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
|
||||
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
|
||||
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
|
||||
reference even on larger Llama-like shapes."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
rows, cols = shape
|
||||
x = torch.randn(rows, cols, dtype=dtype)
|
||||
out = quant_dequant_mxfp4(x)
|
||||
ref = qdq_mxfp4_torch(x, "even")
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == dtype
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
def test_vllm_quant_dequant_mxfp4_is_deterministic():
|
||||
"""The public vLLM MXFP4 QDQ helper should stay bitwise deterministic."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(3)
|
||||
|
||||
x = torch.randn(64, 128, dtype=torch.bfloat16)
|
||||
|
||||
_assert_deterministic(quant_dequant_mxfp4, x, n_runs=4)
|
||||
|
||||
|
||||
# Triton MXFP4 quant tests ------------------------------------------------
|
||||
|
||||
|
||||
def test_aiter_dynamic_mxfp4_quant_output_format():
|
||||
"""dynamic_mxfp4_quant returns packed uint8 FP4 values and E8M0 uint8 scales.
|
||||
|
||||
OCP MXFP4: block_size=32, 2 FP4 E2M1 values packed per byte.
|
||||
Scale shape: (M, K // 32); one E8M0 exponent byte per 32-element block.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = 64, 256
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x)
|
||||
|
||||
# FP4 values packed 2-per-byte; shape (M, K // 2)
|
||||
assert x_fp4.shape == (M, K // 2), (
|
||||
f"Expected fp4 shape ({M}, {K // 2}), got {x_fp4.shape}"
|
||||
)
|
||||
assert x_fp4.dtype == torch.uint8
|
||||
|
||||
# One E8M0 scale byte per 32-element block; shape (M, K // 32)
|
||||
assert x_scale.shape == (M, K // 32), (
|
||||
f"Expected scale shape ({M}, {K // 32}), got {x_scale.shape}"
|
||||
)
|
||||
assert x_scale.dtype == torch.uint8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(128, 4096), # Llama-7B hidden
|
||||
(256, 11008), # Llama-7B FFN
|
||||
(64, 14336), # Llama-70B FFN
|
||||
(512, 8192), # Llama-70B hidden
|
||||
(32, 28672), # DeepSeek-style large FFN
|
||||
],
|
||||
)
|
||||
def test_aiter_dynamic_mxfp4_quant_llama_shapes(shape, dtype):
|
||||
"""MXFP4 quantization output format is correct for Llama-class weight shapes.
|
||||
|
||||
Targets B200 parity: NVIDIA nvfp4 tests parametrize over Llama shapes
|
||||
(7168, 14336, 28672...). This covers gfx950 MXFP4 quant format validation.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = shape
|
||||
x = torch.randn(M, K, dtype=dtype)
|
||||
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x)
|
||||
|
||||
assert x_fp4.shape == (M, K // 2)
|
||||
assert x_fp4.dtype == torch.uint8
|
||||
assert x_scale.shape == (M, K // 32)
|
||||
assert x_scale.dtype == torch.uint8
|
||||
assert not torch.any(torch.isnan(x_fp4.float()))
|
||||
|
||||
|
||||
def test_aiter_dynamic_mxfp4_quant_determinism():
|
||||
"""dynamic_mxfp4_quant is bitwise deterministic across repeated runs."""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(7)
|
||||
x = torch.randn(128, 256, dtype=torch.bfloat16)
|
||||
|
||||
_assert_deterministic(dynamic_mxfp4_quant, x, n_runs=4)
|
||||
|
||||
|
||||
# gfx950 hardware FP4 GEMM tests ------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(64, 128, 64), # small square-ish
|
||||
(128, 256, 128), # medium
|
||||
(128, 4096, 4096), # Llama-7B hidden square
|
||||
(256, 4096, 11008), # Llama-7B FFN
|
||||
(64, 8192, 28672), # Llama-70B FFN
|
||||
],
|
||||
)
|
||||
def test_aiter_fp4_gemm_a4w4_accuracy(shape):
|
||||
"""AITER A4W4 FP4 GEMM output is close to matmul on dequantized weights.
|
||||
|
||||
Tests gfx950 parity with B200 test_nvfp4_scaled_mm.py: E2M1 x E2M1 GEMM
|
||||
with block-scaled quantization, multiple Llama-class shapes.
|
||||
Requires gfx950 hardware; gracefully skips elsewhere.
|
||||
"""
|
||||
atol = GEMM_ATOL
|
||||
rtol = GEMM_RTOL
|
||||
pass_rate = A4W4_GEMM_PASS_RATES[shape]
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
M, K, N = shape
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16) # weight stored as (N, K)
|
||||
|
||||
# Quantize
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
# FP4 GEMM computes A_fp4 (M, K) @ B_fp4.T (K, N) into (M, N).
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N), f"Expected ({M}, {N}), got {out.shape}"
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
# Reference: matmul on dequantized FP4 inputs (quark roundtrip)
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float())
|
||||
|
||||
# FP4 has only 3 mantissa bits; large K amplifies the accumulation error.
|
||||
# These budgets were measured on gfx950 and keep p99/p999 drift visible
|
||||
# without pretending the kernel is close to BF16 matmul everywhere.
|
||||
_print_close_stats(
|
||||
f"a4w4_gemm shape={shape}",
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
|
||||
)
|
||||
_assert_accurate(
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
PRESHUFFLED_SHAPES,
|
||||
)
|
||||
def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
"""The preshuffled FP4 GEMM path should run on tuned shapes with the same
|
||||
packed inputs vLLM uses in production.
|
||||
|
||||
This kernel only advertises a narrow tuned `(N, K)` set on gfx950. The
|
||||
important contract here is that those tuned shapes execute with the
|
||||
production-style shuffled weights and stay deterministic.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter import per_1x32_f4_quant_hip
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import (
|
||||
gemm_afp4wfp4_preshuffle,
|
||||
)
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(1)
|
||||
M, K, N = shape
|
||||
|
||||
assert M <= 64
|
||||
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
|
||||
# must agree with the config the kernel itself resolves.
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
|
||||
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
|
||||
|
||||
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
A_q, A_s = per_1x32_f4_quant_hip(A, shuffle=M >= 32)
|
||||
|
||||
scale_rows, scale_cols = B_scale.shape
|
||||
B_scale = (
|
||||
B_scale.view(scale_rows // 32, 2, 16, scale_cols // 8, 2, 4, 1)
|
||||
.permute(0, 3, 5, 2, 4, 1, 6)
|
||||
.contiguous()
|
||||
.view(scale_rows, scale_cols)
|
||||
)
|
||||
B_fp4 = shuffle_weight(B_fp4, layout=(16, 16))
|
||||
|
||||
if M >= 32:
|
||||
A_s = A_s.contiguous().view(torch.uint8).reshape(A_s.shape[0] // 32, -1)
|
||||
else:
|
||||
A_s = A_s[:M, ...].contiguous().view(torch.uint8)
|
||||
B_scale = B_scale.contiguous().view(torch.uint8).reshape(B_scale.shape[0] // 32, -1)
|
||||
|
||||
def run_preshuffled() -> torch.Tensor:
|
||||
y = torch.empty(M, N, device="cuda", dtype=torch.bfloat16)
|
||||
return gemm_afp4wfp4_preshuffle(
|
||||
A_q.contiguous().view(torch.uint8),
|
||||
B_fp4.contiguous().view(torch.uint8).reshape(B_fp4.shape[0] // 16, -1),
|
||||
A_s,
|
||||
B_scale,
|
||||
torch.bfloat16,
|
||||
y,
|
||||
)
|
||||
|
||||
out = run_preshuffled()
|
||||
assert out.shape == (M, N)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
_assert_deterministic(run_preshuffled, n_runs=3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
|
||||
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
|
||||
has no tuned preshuffled config even though twice its K does."""
|
||||
_assert_aiter_supported()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
|
||||
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_gemm_a4w4_determinism():
|
||||
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
|
||||
|
||||
Targets parity with B200 FP4 determinism requirements.
|
||||
Requires gfx950.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(2)
|
||||
|
||||
M, K, N = 128, 256, 128
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
def run_gemm():
|
||||
return gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
_assert_deterministic(run_gemm, n_runs=4)
|
||||
|
||||
|
||||
# gfx950 hardware FP4 dynamic quantization --------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(128, 256),
|
||||
(256, 4096),
|
||||
(64, 14336),
|
||||
],
|
||||
)
|
||||
def test_aiter_hardware_fp4_dynamic_quant_format(shape):
|
||||
"""Aiter hardware FP4 dynamic quant produces correct output format.
|
||||
|
||||
Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1).
|
||||
Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output.
|
||||
Requires gfx950.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
|
||||
from aiter import dynamic_per_group_scaled_quant_fp4
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = shape
|
||||
group_size = 32
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
out_fp4 = torch.empty(M, K // 2, dtype=torch.uint8)
|
||||
scales = torch.empty(M, K // group_size, dtype=torch.uint8)
|
||||
|
||||
dynamic_per_group_scaled_quant_fp4(out_fp4, x, scales, group_size)
|
||||
|
||||
assert out_fp4.shape == (M, K // 2), (
|
||||
f"Shape {shape}: expected fp4 ({M}, {K // 2}), got {out_fp4.shape}"
|
||||
)
|
||||
assert scales.shape == (M, K // group_size), (
|
||||
f"Shape {shape}: expected scale ({M}, {K // group_size}), got {scales.shape}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_hardware_fp4_quant_vs_triton():
|
||||
"""Hardware FP4 quant should dequantize back to the same MXFP4 values as
|
||||
the Triton path.
|
||||
|
||||
The packed bytes are not identical on gfx950, but with
|
||||
``shuffle_scale=False`` the dequantized results match tightly.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
|
||||
from aiter import dynamic_per_group_scaled_quant_fp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(5)
|
||||
|
||||
M, K = 128, 256
|
||||
group_size = 32
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
|
||||
# Hardware path
|
||||
out_hw = torch.empty(M, K // 2, dtype=torch.uint8)
|
||||
scales_hw = torch.empty(M, K // group_size, dtype=torch.uint8)
|
||||
dynamic_per_group_scaled_quant_fp4(
|
||||
out_hw, x, scales_hw, group_size, shuffle_scale=False
|
||||
)
|
||||
|
||||
# Triton path
|
||||
out_triton, scales_triton = dynamic_mxfp4_quant(x)
|
||||
|
||||
assert out_hw.shape == out_triton.shape
|
||||
assert scales_hw.shape == scales_triton.shape
|
||||
|
||||
dq_hw = dq_mxfp4_torch(out_hw, scales_hw, torch.bfloat16)
|
||||
dq_triton = dq_mxfp4_torch(out_triton, scales_triton, torch.bfloat16)
|
||||
|
||||
_print_close_stats(
|
||||
"hardware_quant_vs_triton",
|
||||
dq_hw.float(),
|
||||
dq_triton.float(),
|
||||
atol=0.25,
|
||||
rtol=0.0,
|
||||
)
|
||||
torch.testing.assert_close(dq_hw, dq_triton, atol=0.25, rtol=0.0)
|
||||
|
||||
|
||||
# ROCm skinny GEMM FP4 tests ----------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize("M", [1, 2, 4, 8]) # decode / skinny batch sizes
|
||||
@pytest.mark.parametrize("N, K", [(4096, 4096), (4096, 11008), (8192, 8192)])
|
||||
def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
|
||||
"""FP4 GEMM accuracy for skinny (small-M) shapes (decode phase).
|
||||
|
||||
Skinny GEMMs (M=1..8) are the bottleneck in decode phase.
|
||||
Tests parity with B200 nvfp4_scaled_mm tests at decode batch sizes.
|
||||
Requires gfx950.
|
||||
"""
|
||||
atol = GEMM_ATOL
|
||||
rtol = GEMM_RTOL
|
||||
pass_rate = SKINNY_GEMM_PASS_RATES[(N, K)]
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
# Accumulate in FP32, then match the GEMM's BF16 output rounding.
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float()).to(torch.bfloat16).float()
|
||||
|
||||
_print_close_stats(
|
||||
f"skinny_gemm M={M} N={N} K={K}",
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
|
||||
)
|
||||
_assert_accurate(
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "supported"),
|
||||
[
|
||||
((2560, 80), True), # shared_expert gate/up_proj (in=2560 -> sn=80)
|
||||
((1280, 80), True), # gate/up_proj under TP=2 (out sharded)
|
||||
((2560, 8), True), # minimal aligned columns
|
||||
((32, 8), True), # minimal aligned rows and columns
|
||||
((2560, 20), False), # shared_expert down_proj (in=640 -> sn=20)
|
||||
((2560, 10), False), # down_proj under TP=2 (in 640->320 -> sn=10)
|
||||
((30, 8), False), # rows not a multiple of 32
|
||||
],
|
||||
)
|
||||
def test_asm_fp4_scale_swizzle_supported_shape_rules(shape, supported):
|
||||
"""The ASM swizzle needs rows % 32 == 0 and columns % 8 == 0."""
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
_asm_fp4_scale_swizzle_supported,
|
||||
)
|
||||
|
||||
weight_scale = torch.empty(shape, dtype=torch.uint8, device="cpu")
|
||||
assert _asm_fp4_scale_swizzle_supported(weight_scale) is supported
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_asm_fp4_scale_swizzle_rejects_non_2d():
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import (
|
||||
_asm_fp4_scale_swizzle_supported,
|
||||
)
|
||||
|
||||
assert not _asm_fp4_scale_swizzle_supported(
|
||||
torch.empty(2560, dtype=torch.uint8, device="cpu")
|
||||
)
|
||||
assert not _asm_fp4_scale_swizzle_supported(
|
||||
torch.empty(80, 2, 16, dtype=torch.uint8, device="cpu")
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
|
||||
[
|
||||
(True, True, (1280, 4096), (1280, 256), True),
|
||||
(True, False, (1280, 4096), (1280, 256), False),
|
||||
(False, True, (1280, 4096), (1280, 256), False),
|
||||
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
|
||||
(True, True, (2560, 160), (2560, 10), False),
|
||||
],
|
||||
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
|
||||
)
|
||||
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
|
||||
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
|
||||
):
|
||||
"""A layer takes the ASM layout only when aiter has tuned configs for both
|
||||
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
|
||||
Otherwise it stays on the Triton layout."""
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
|
||||
lambda n, k: preshuffle_tuned,
|
||||
)
|
||||
kernel = object.__new__(AiterMxfp4LinearKernel)
|
||||
kernel.asm_gemm_supported = True
|
||||
kernel.use_asm_gemm = False
|
||||
kernel.out_dtype = torch.bfloat16
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = Parameter(
|
||||
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = Parameter(
|
||||
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.use_asm_gemm is expected
|
||||
if not expected:
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
@@ -1,286 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Native CDNA4 MXFP8 linear (``RocmDotScaledMxfp8LinearKernel``).
|
||||
|
||||
DeepSeek V4.1 routes every attention linear through this kernel. It used to
|
||||
carry two BF16-dequantize workarounds on ROCm (one for all ``.attn.`` linears,
|
||||
one for the row-parallel ``wo_b``, whose comment blamed NaNs at small token
|
||||
counts). These tests pin the two properties those workarounds stood in for --
|
||||
finite output and MXFP8-level accuracy -- across the V4.1 attention shapes at
|
||||
TP=4 and the small token counts decode actually runs.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
_mxfp8_e4m3_quantize_torch,
|
||||
_mxfp8_e4m3_quantize_triton,
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not current_platform.supports_mx(),
|
||||
reason="native MX linear requires CDNA4 (gfx95x)",
|
||||
)
|
||||
|
||||
# (name, N, K) for the DeepSeek V4.1 attention linears at tensor_parallel=4:
|
||||
# fused wq_a|wkv is replicated, wq_b/wo_a are column-parallel, wo_b is
|
||||
# row-parallel (so K, not N, is sharded).
|
||||
V41_ATTN_SHAPES = [
|
||||
("fused_wqa_wkv", 1280 + 512, 5120),
|
||||
("wq_b", 64 * 512 // 4, 1280),
|
||||
("wo_a", 8 * 1024 // 4, 4096),
|
||||
("wo_b", 5120, 8192 // 4),
|
||||
]
|
||||
|
||||
|
||||
def _make_layer(n: int, k: int, device: str) -> torch.nn.Module:
|
||||
weight_bf16 = torch.randn(n, k, device=device, dtype=torch.bfloat16) * 0.05
|
||||
weight, weight_scale = _mxfp8_e4m3_quantize_torch(weight_bf16)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,n,k", V41_ATTN_SHAPES, ids=lambda v: str(v))
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 7, 16, 31, 32, 128, 1024])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_matches_dequant_reference(name, n, k, num_tokens):
|
||||
"""Output is finite and matches an exact MXFP8 GEMM of the same inputs.
|
||||
|
||||
The reference quantizes the activation with the same quantizer and then
|
||||
contracts in FP32, so the tolerance covers only kernel/accumulation error,
|
||||
not the MXFP8 quantization error both sides share.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
torch.manual_seed(num_tokens)
|
||||
device = "cuda"
|
||||
layer = _make_layer(n, k, device)
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
|
||||
out = kernel.apply_weights(layer, x)
|
||||
|
||||
assert out.shape == (num_tokens, n)
|
||||
assert torch.isfinite(out).all(), f"{name}: non-finite output"
|
||||
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
expected = (
|
||||
dequant_mxfp8_to_bf16(x_q, x_scale).float()
|
||||
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
|
||||
)
|
||||
rel = (out.float() - expected).norm() / expected.norm()
|
||||
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_handles_extreme_magnitudes():
|
||||
"""Rows that are all-zero or near the E4M3 range must not produce NaNs.
|
||||
|
||||
A degenerate block makes the dynamic activation quantizer pick an extreme
|
||||
E8M0 scale; a zero row makes it pick the tiny-clamped one. Both feed
|
||||
``tl.dot_scaled`` directly.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
n, k = 5120, 2048 # wo_b at TP=4
|
||||
layer = _make_layer(n, k, device)
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(8, k, device=device, dtype=torch.bfloat16)
|
||||
x[0] = 0.0
|
||||
x[1] = 1e-30
|
||||
x[2] = 400.0
|
||||
x[3, ::2] = 0.0
|
||||
|
||||
out = kernel.apply_weights(layer, x)
|
||||
assert torch.isfinite(out).all()
|
||||
assert (out[0] == 0).all(), "an all-zero activation row must give a zero row"
|
||||
|
||||
|
||||
# (name, N, K) of the DeepSeek-V4.1-Flash MXFP8 linears at TP=2 and TP=4. The
|
||||
# checkpoint stores their scales in 32x32 blocks; shared_down at TP=4 has
|
||||
# K % 128 != 0, which only the block-scaled GEMM runs natively.
|
||||
V41_BLOCK32_SHAPES = [
|
||||
("fused_wqa_wkv", 1280 + 512, 5120),
|
||||
("wq_b_tp2", 64 * 512 // 2, 1280),
|
||||
("indexer_wq_b", 32 * 128, 1280),
|
||||
("wo_b_tp2", 5120, 8192 // 2),
|
||||
("shared_gate_up_tp2", 2 * 2304 // 2, 5120),
|
||||
("shared_down_tp2", 5120, 2304 // 2),
|
||||
("shared_down_tp4", 5120, 2304 // 4),
|
||||
]
|
||||
|
||||
|
||||
def _make_block32_layer(n: int, k: int, device: str) -> torch.nn.Module:
|
||||
"""An MXFP8 layer with 32x32 block scales, expanded per row as loaded."""
|
||||
blocks = (torch.randn(n, k, device=device) * 0.05).view(n // 32, 32, k // 32, 32)
|
||||
amax = blocks.abs().amax(dim=(1, 3)).clamp(min=torch.finfo(torch.float32).tiny)
|
||||
bits = (torch.ceil(torch.log2(amax / 448.0)) + 127.0).clamp(0, 254)
|
||||
weight = (blocks * torch.exp2(127.0 - bits)[:, None, :, None]).view(n, k)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
weight.to(torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
bits.to(torch.uint8).repeat_interleave(32, dim=0), requires_grad=False
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,n,k", V41_BLOCK32_SHAPES, ids=lambda v: str(v))
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 33, 192, 1024])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_block32_matches_dequant_reference(name, n, k, num_tokens):
|
||||
"""32x32 block scales stay compact and the block-scaled GEMM is exact.
|
||||
|
||||
The token counts cover the packed, tiled and split-K configurations.
|
||||
"""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
torch.manual_seed(num_tokens)
|
||||
device = "cuda"
|
||||
layer = _make_block32_layer(n, k, device)
|
||||
row_scale = layer.weight_scale.data.clone()
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
assert layer.weight_scale.shape == (n // 32, k // 32)
|
||||
|
||||
x = torch.randn(num_tokens, k, device=device, dtype=torch.bfloat16) * 0.5
|
||||
out = kernel.apply_weights(layer, x)
|
||||
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(x)
|
||||
expected = (
|
||||
dequant_mxfp8_to_bf16(x_q, x_scale).float()
|
||||
@ dequant_mxfp8_to_bf16(layer.weight, row_scale).float().T
|
||||
)
|
||||
assert out.shape == (num_tokens, n)
|
||||
rel = (out.float() - expected).norm() / expected.norm()
|
||||
assert rel < 5e-3, f"{name}: relative error {rel:.4f}"
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_linear_keeps_per_row_scales():
|
||||
"""Scales that differ within a 32-row block are not compacted."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
n, k = 64, 256
|
||||
layer = _make_block32_layer(n, k, "cuda")
|
||||
layer.weight_scale.data[1, 0] += 1
|
||||
kernel = RocmDotScaledMxfp8LinearKernel(Mxfp8LinearLayerConfig(weight_shape=(n, k)))
|
||||
kernel.process_weights_after_loading(layer)
|
||||
assert layer.weight_scale.shape == (n, k // 32)
|
||||
|
||||
|
||||
def _split_k_case(packed: bool):
|
||||
"""A 3-way in-launch split-K on shared_down TP4, with K and M tails."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import rocm_block32_gemm as g
|
||||
|
||||
n, k, m = 5120, 576, 19
|
||||
layer = _make_block32_layer(n, k, "cuda")
|
||||
x_q, x_scale = mxfp8_e4m3_quantize(
|
||||
torch.randn(m, k, device="cuda", dtype=torch.bfloat16)
|
||||
)
|
||||
expected = (
|
||||
dequant_mxfp8_to_bf16(x_q, x_scale).float()
|
||||
@ dequant_mxfp8_to_bf16(layer.weight, layer.weight_scale).float().T
|
||||
)
|
||||
if packed:
|
||||
cfg = g._packed(16, 32, 128, 1, 2, 2, 16, 0, 3)
|
||||
else:
|
||||
cfg = g._tiled(32, 64, 128, 1, True, 4, 2, 16, 0, 3)
|
||||
args = (x_q, x_scale, layer.weight, layer.weight_scale.data[::32].contiguous())
|
||||
return g, cfg, args, expected
|
||||
|
||||
|
||||
def _rel_err(out: torch.Tensor, expected: torch.Tensor) -> float:
|
||||
return ((out.float() - expected).norm() / expected.norm()).item()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_block32_split_k_in_launch(packed):
|
||||
"""Splits summed by the tile's last program are exact and repeatable."""
|
||||
torch.manual_seed(0)
|
||||
g, cfg, args, expected = _split_k_case(packed)
|
||||
out = torch.empty_like(expected, dtype=torch.bfloat16)
|
||||
g._launch(*args, out, cfg)
|
||||
assert _rel_err(out, expected) < 5e-3
|
||||
first = out.clone()
|
||||
for _ in range(20):
|
||||
g._launch(*args, out, cfg)
|
||||
assert torch.equal(out, first)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("packed", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_block32_split_k_first_use_in_capture(packed):
|
||||
"""A stream whose first split-K launch is being captured has no counters
|
||||
yet; it must fall back to an unfused split instead of failing."""
|
||||
torch.manual_seed(0)
|
||||
g, cfg, args, expected = _split_k_case(packed)
|
||||
out = torch.zeros_like(expected, dtype=torch.bfloat16)
|
||||
stream = torch.cuda.Stream()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph, stream=stream):
|
||||
g._launch(*args, out, cfg)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert _rel_err(out, expected) < 5e-3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_rows", [1, 64, 65])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_mxfp8_quantizer_matches_torch_on_degenerate_blocks(num_rows):
|
||||
"""The fused ROCm quantizer must agree with the torch one, zeros included.
|
||||
|
||||
An all-zero 32-element block clamps the E8M0 scale to 0, i.e. 2**-127 --
|
||||
subnormal in fp32 and flushed to zero on CDNA. Dividing by it turned the
|
||||
block into 0/0 == NaN, which then poisoned every output column of the GEMM
|
||||
that consumed it.
|
||||
"""
|
||||
torch.manual_seed(num_rows)
|
||||
device = "cuda"
|
||||
x = torch.randn(num_rows, 128, device=device, dtype=torch.bfloat16)
|
||||
x[0] = 0.0 # all-zero row: every block clamps to scale byte 0
|
||||
x[-1, :32] = 0.0 # single zero block in an otherwise normal row
|
||||
x[-1, 32:64] = 1e-38 # denormal-ish block
|
||||
|
||||
q_triton, s_triton = _mxfp8_e4m3_quantize_triton(x)
|
||||
q_torch, s_torch = _mxfp8_e4m3_quantize_torch(x)
|
||||
|
||||
assert not torch.isnan(q_triton.float()).any()
|
||||
torch.testing.assert_close(s_triton, s_torch, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
q_triton.view(torch.uint8), q_torch.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
@@ -1,416 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import ref_dynamic_per_tensor_fp8_quant
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
# Global per-test cleanup costs more than the tests themselves.
|
||||
# These tests can just cleanup once at the end
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
SEEDS = [0]
|
||||
|
||||
# These options are independent in the kernels, so the sets below cover every
|
||||
# pair of them instead of the full product. bias_mode: 0 = none, 1 = (m,),
|
||||
# 2 = (n, m).
|
||||
OPTIONS_WVSPLITKRC = [
|
||||
# dtype, padded_a, bias_mode, xnorm
|
||||
(torch.float16, False, 0, False),
|
||||
(torch.float16, True, 1, True),
|
||||
(torch.float16, True, 2, False),
|
||||
(torch.bfloat16, True, 0, True),
|
||||
(torch.bfloat16, False, 1, False),
|
||||
(torch.bfloat16, False, 2, True),
|
||||
]
|
||||
|
||||
OPTIONS_WVSPLITK = [
|
||||
# dtype, padded_a, padded_b, bias_mode, xnorm
|
||||
(torch.float16, False, False, 0, False),
|
||||
(torch.float16, False, True, 1, True),
|
||||
(torch.float16, True, False, 2, True),
|
||||
(torch.bfloat16, True, True, 0, True),
|
||||
(torch.bfloat16, True, False, 1, False),
|
||||
(torch.bfloat16, False, True, 2, False),
|
||||
]
|
||||
|
||||
OPTIONS_WVSPLITK_FP8 = [
|
||||
# dtype, padded_a, padded_b, biased, xnorm
|
||||
(torch.float16, False, False, True, True),
|
||||
(torch.bfloat16, True, False, False, True),
|
||||
(torch.bfloat16, False, True, True, True),
|
||||
(torch.float16, True, False, True, False),
|
||||
(torch.float16, False, True, False, False),
|
||||
(torch.bfloat16, True, True, False, False),
|
||||
]
|
||||
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
|
||||
# kWvSlots in csrc/rocm/skinny_gemms.cu; beyond it streams take overflow
|
||||
# workspaces.
|
||||
WVSPLITKRC_SLOTS = 8
|
||||
|
||||
# The CU count is the grid size. Production passes num_compute_units(); a small
|
||||
# value manufactures better concurrency for these tests.
|
||||
WVSPLITKRC_TEST_CU = 16
|
||||
|
||||
# Gate spin, ~125 ms on gfx950 -- must outlast the host enqueue loop it gates.
|
||||
_GATE_CYCLES = 300_000_000
|
||||
|
||||
# Specific (N, K, M) combinations for targeted testing
|
||||
NKM_FACTORS_LLMM1 = [
|
||||
# Small, medium, large cases
|
||||
(1, 8, 16),
|
||||
(1, 32, 64),
|
||||
(1, 128, 256),
|
||||
(1, 512, 1024),
|
||||
(1, 2048, 4096),
|
||||
# Edge cases with specific K sizes
|
||||
(1, 6144, 1024),
|
||||
(1, 8192, 2048),
|
||||
# Very large case
|
||||
(1, 4096, 8192),
|
||||
]
|
||||
|
||||
NKM_FACTORS_WVSPLITK = [
|
||||
# Different batch sizes with key dimensions
|
||||
(1, 32, 16),
|
||||
(1, 64, 64),
|
||||
(2, 256, 256),
|
||||
(3, 1024, 1024),
|
||||
(4, 4096, 4096),
|
||||
(4, 4096, 4096 + 1),
|
||||
(4, 4096 + 16, 4096),
|
||||
(4, 4096 + 16, 4096 + 1),
|
||||
# Extended K values
|
||||
(1, 9216, 512),
|
||||
(2, 10240, 1024),
|
||||
(4, 16384, 8192),
|
||||
(4, 16384 * 2, 8192),
|
||||
(4, 16384 * 2, 8192 + 1),
|
||||
(4, 16384 * 2 + 16, 8192),
|
||||
(4, 16384 * 2 + 16, 8192 + 1),
|
||||
# Minimum M constraint validation (m >= 8)
|
||||
(1, 64, 8),
|
||||
(2, 128, 8),
|
||||
(4, 256, 8),
|
||||
]
|
||||
|
||||
# N is bucketed up to its next ^2 (16/32/64/128) and the remainder is masked
|
||||
N_FACTORS_WVSPLITKRC = [
|
||||
13,
|
||||
16,
|
||||
17,
|
||||
25,
|
||||
29,
|
||||
31,
|
||||
32,
|
||||
41,
|
||||
51,
|
||||
64,
|
||||
71,
|
||||
81,
|
||||
91,
|
||||
103,
|
||||
117,
|
||||
128,
|
||||
]
|
||||
# K shards are 512 wide, evenly divided or not, +8 for a partial 8-element load
|
||||
K_FACTORS_WVSPLITKRC = [2880, 2880 + 8, 3072, 3072 + 8]
|
||||
# M tiles are 64 rows, +16 for a partial tile
|
||||
M_FACTORS_WVSPLITKRC = [128, 128 + 16, 256, 256 + 16, 640, 640 + 16]
|
||||
|
||||
# (N, K, M) with more K-shards than the readback can stage in one LDS pass.
|
||||
# Spans both CHUNKK values and both N-tile counts.
|
||||
NKM_FACTORS_WVSPLITKRC_LARGE_K = [
|
||||
(128, 6144, 128),
|
||||
(96, 8192, 128),
|
||||
# Keep 24 K-shards (batched readback) within the DPX budget of 128 CUs.
|
||||
(128, 12288, 64),
|
||||
(32, 12288, 128),
|
||||
]
|
||||
|
||||
NKM_FACTORS_WVSPLITK_FP8 = [
|
||||
# FP8-specific cases with K % 16 == 0
|
||||
(1, 16, 16),
|
||||
(1, 32, 16 + 16),
|
||||
(1, 64, 64),
|
||||
(1, 64, 64 + 16),
|
||||
(1, 64 + 16, 64),
|
||||
(1, 64 + 16, 64 + 16),
|
||||
(4, 64, 64),
|
||||
(4, 64, 64 + 16),
|
||||
(4, 64 + 16, 64),
|
||||
(4, 64 + 16, 64 + 16),
|
||||
(2, 512, 512),
|
||||
(3, 512, 512),
|
||||
(3, 512, 512 + 16),
|
||||
(4, 512, 512),
|
||||
(3, 2048, 2048),
|
||||
(3, 2048, 2048 + 16),
|
||||
(4, 2048 + 16, 2048),
|
||||
(4, 2048 + 16, 2048 + 16),
|
||||
(4, 4096, 4096),
|
||||
(4, 16400, 2048),
|
||||
(4, 16400, 2048 + 16),
|
||||
# Extended FP8 dimensions not covered by WVSPLITK
|
||||
(1, 14336, 1024),
|
||||
(2, 24576, 2048),
|
||||
(4, 32768, 28672),
|
||||
(4, 32768 * 2, 28672),
|
||||
(4, 32768 * 2, 28672 + 16),
|
||||
(4, 32768 * 2 + 16, 28672),
|
||||
(4, 32768 * 2 + 16, 28672 + 16),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def cleanup_after_all_tests():
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
def pad_fp8(weight):
|
||||
num_pad = 256 // weight.element_size()
|
||||
import torch.nn.functional as F
|
||||
|
||||
return F.pad(weight, (0, num_pad), "constant", 0)[..., :-num_pad]
|
||||
|
||||
|
||||
def make_bias(bias_mode, n, m, dtype):
|
||||
if bias_mode == 0:
|
||||
return None
|
||||
shape = (m,) if bias_mode == 1 else (n, m)
|
||||
return torch.rand(shape, dtype=dtype, device="cuda") * 2 - 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", N_FACTORS_WVSPLITKRC)
|
||||
@pytest.mark.parametrize("k", K_FACTORS_WVSPLITKRC)
|
||||
@pytest.mark.parametrize("m", M_FACTORS_WVSPLITKRC)
|
||||
@pytest.mark.parametrize("dtype,padded_a,bias_mode,xnorm", OPTIONS_WVSPLITKRC)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
|
||||
def test_rocm_wvsplitkrc_kernel(n, k, m, dtype, padded_a, bias_mode, xnorm, seed):
|
||||
torch.manual_seed(seed)
|
||||
cu_count = num_compute_units()
|
||||
|
||||
# Next ^2 of n
|
||||
N_p2 = 1 << (n - 1).bit_length()
|
||||
# With 64 Ms per CU (each of 4 SIMDs working on a 16x16 tile),
|
||||
# and each working on a 512-shard of K, how many CUs would we need?
|
||||
rndup_cus = ((m + 64 - 1) // 64) * ((k + 512 - 1) // 512)
|
||||
# How many of 4 waves in a group can work on same 16 Ms at same time?
|
||||
# This reduces the Ms each group works on, i.e. increasing the number of CUs needed.
|
||||
GrpsShrB = min(N_p2 // 16, 4)
|
||||
# Given the above, how many CUs would we need?
|
||||
CuNeeded = rndup_cus * GrpsShrB
|
||||
# Deterministic reduction stores one float workspace value per K shard.
|
||||
fits_wvsplitkrc = (N_p2 * m * ((k + 512 - 1) // 512)) <= 128 * 1024 * 12
|
||||
fits_wvsplitkrc &= CuNeeded <= cu_count
|
||||
|
||||
if not fits_wvsplitkrc:
|
||||
pytest.skip("Too large for wvSplitKrc")
|
||||
|
||||
xavier = (
|
||||
math.sqrt(2 / k) if xnorm else 1
|
||||
) # normalize to avoid large output-bias deltas
|
||||
A = torch.randn(n, k, dtype=dtype, device="cuda") * xavier
|
||||
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
|
||||
if padded_a:
|
||||
A = pad_fp8(A)
|
||||
|
||||
BIAS = make_bias(bias_mode, n, m, dtype)
|
||||
|
||||
ref_out = torch.nn.functional.linear(A, B, BIAS)
|
||||
out = ops.wvSplitKrc(A, B, cu_count, BIAS)
|
||||
|
||||
if xnorm:
|
||||
# The O(1) bias lifts outputs to ~O(1), where one bf16 ULP (~3.9e-3, the
|
||||
# worst measured divergence) exceeds 1e-3. Bump atol to 5e-3 only for
|
||||
# biased bf16 (above that ULP, still under finfo(bf16).eps); keep 1e-3
|
||||
# otherwise.
|
||||
atol = 5e-3 if (dtype == torch.bfloat16 and BIAS is not None) else 1e-3
|
||||
torch.testing.assert_close(out, ref_out, atol=atol, rtol=1e-8)
|
||||
else:
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITKRC_LARGE_K)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
|
||||
def test_rocm_wvsplitkrc_large_k(n, k, m, dtype, seed):
|
||||
"""K large enough that the split-K readback must stage LDS in batches."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
xavier = math.sqrt(2 / k)
|
||||
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
|
||||
ref_out = torch.nn.functional.linear(A, B, None)
|
||||
out = ops.wvSplitKrc(A, B, num_compute_units(), None)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_streams", [WVSPLITKRC_SLOTS, WVSPLITKRC_SLOTS + 4])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="only meant for gfx950")
|
||||
def test_rocm_wvsplitkrc_multistream(n_streams, dtype, seed):
|
||||
"""Concurrent streams must not share a split-K workspace.
|
||||
|
||||
The workspace is cleared by protocol rather than per invocation, so sharing
|
||||
one corrupts the reduction and returns wrong numbers rather than failing.
|
||||
n_streams > WVSPLITKRC_SLOTS additionally exercises the overflow path.
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
n, k, m = 16, 2048, 256
|
||||
iters = 4
|
||||
xavier = math.sqrt(2 / k)
|
||||
|
||||
B = torch.randn(m, k, dtype=dtype, device="cuda") * xavier
|
||||
As = [
|
||||
torch.randn(n, k, dtype=dtype, device="cuda") * xavier for _ in range(n_streams)
|
||||
]
|
||||
refs = [torch.nn.functional.linear(A, B, None) for A in As]
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
streams = [torch.Stream() for _ in range(n_streams)]
|
||||
outs: list[list[torch.Tensor]] = [[] for _ in range(n_streams)]
|
||||
_release_together(streams)
|
||||
# Round-robin the enqueues so kernels from different streams are in flight
|
||||
# at the same time.
|
||||
for _ in range(iters):
|
||||
for i, s in enumerate(streams):
|
||||
with torch.cuda.stream(s):
|
||||
outs[i].append(ops.wvSplitKrc(As[i], B, WVSPLITKRC_TEST_CU, None))
|
||||
for s in streams:
|
||||
torch.accelerator.current_stream().wait_stream(s)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for i in range(n_streams):
|
||||
for out in outs[i]:
|
||||
torch.testing.assert_close(out, refs[i], atol=1e-3, rtol=1e-2)
|
||||
|
||||
|
||||
def _release_together(streams):
|
||||
"""Release every stream at once, once everything below is enqueued.
|
||||
|
||||
An event recorded on an idle stream is already complete, so waiting on it
|
||||
gates nothing -- the spin, not the event, is what holds the streams.
|
||||
"""
|
||||
blocker = torch.Stream()
|
||||
with torch.cuda.stream(blocker):
|
||||
torch.cuda._sleep(_GATE_CYCLES)
|
||||
gate = torch.Event()
|
||||
gate.record()
|
||||
for s in streams:
|
||||
s.wait_event(gate)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_LLMM1)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("rows_per_block", [2, 4, 8, 16])
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
@torch.inference_mode()
|
||||
def test_rocm_llmm1_kernel(n, k, m, dtype, rows_per_block, seed):
|
||||
torch.manual_seed(seed)
|
||||
# TODO: Zero-centering the inputs causes errors for LLMM1!
|
||||
# Without that the numbers quickly saturate, and may
|
||||
# be giving false matches.
|
||||
A = torch.rand(n, k, dtype=dtype, device="cuda")
|
||||
B = torch.rand(m, k, dtype=dtype, device="cuda")
|
||||
|
||||
ref_out = torch.matmul(A, B.t())
|
||||
out = ops.LLMM1(B, A, rows_per_block)
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-8, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITK)
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,padded_a,padded_b,bias_mode,xnorm",
|
||||
OPTIONS_WVSPLITK,
|
||||
)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="only test for rocm")
|
||||
def test_rocm_wvsplitk_kernel(
|
||||
n, k, m, dtype, padded_a, padded_b, bias_mode, xnorm, seed
|
||||
):
|
||||
torch.manual_seed(seed)
|
||||
cu_count = num_compute_units()
|
||||
|
||||
xavier = (
|
||||
math.sqrt(2 / k) if xnorm else 1
|
||||
) # normalize to avoid large output-bias deltas
|
||||
A = (torch.rand(n, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
B = (torch.rand(m, k, dtype=dtype, device="cuda") * 2 - 1) * xavier
|
||||
|
||||
BIAS = make_bias(bias_mode, n, m, dtype)
|
||||
|
||||
if padded_a:
|
||||
A = pad_fp8(A)
|
||||
if padded_b:
|
||||
B = pad_fp8(B)
|
||||
|
||||
ref_out = torch.nn.functional.linear(A, B, BIAS)
|
||||
out = ops.wvSplitK(B, A.view(-1, A.size(-1)), cu_count, BIAS)
|
||||
|
||||
# Accumulation error in fp16 GEMM scales with sqrt(K)
|
||||
atol = torch.finfo(dtype).eps * math.sqrt(k)
|
||||
torch.testing.assert_close(out, ref_out, atol=atol, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,m", NKM_FACTORS_WVSPLITK_FP8)
|
||||
@pytest.mark.parametrize(
|
||||
"dtype,padded_a,padded_b,biased,xnorm",
|
||||
OPTIONS_WVSPLITK_FP8,
|
||||
)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and current_platform.supports_fp8()),
|
||||
reason="only test for rocm fp8",
|
||||
)
|
||||
def test_rocm_wvsplitk_fp8_kernel(
|
||||
n, k, m, dtype, padded_a, padded_b, biased, xnorm, seed
|
||||
):
|
||||
torch.manual_seed(seed)
|
||||
|
||||
xavier = math.sqrt(2 / k) if xnorm else 1 # normalize to avoid large deltas
|
||||
A = (torch.rand(n, k, device="cuda") * 2 - 1) * xavier
|
||||
B = (torch.rand(m, k, device="cuda") * 2 - 1) * xavier
|
||||
|
||||
A, scale_a = ref_dynamic_per_tensor_fp8_quant(A)
|
||||
B, scale_b = ref_dynamic_per_tensor_fp8_quant(B)
|
||||
if padded_b:
|
||||
B = pad_fp8(B)
|
||||
if padded_a:
|
||||
A = pad_fp8(A)
|
||||
|
||||
BIAS = None if (not biased) else (torch.rand(m, dtype=dtype, device="cuda") * 2 - 1)
|
||||
|
||||
ref_out = torch._scaled_mm(
|
||||
A, B.t(), out_dtype=dtype, scale_a=scale_a, scale_b=scale_b, bias=BIAS
|
||||
)
|
||||
out = ops.wvSplitKQ(B, A, dtype, scale_a, scale_b, num_compute_units(), BIAS)
|
||||
|
||||
if xnorm:
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-3, rtol=1e-8)
|
||||
elif k >= 32 * 1024:
|
||||
# wider pytrch thresh for large-K & no xnorm
|
||||
torch.testing.assert_close(out, ref_out, atol=0.07, rtol=5e-2)
|
||||
else:
|
||||
torch.testing.assert_close(out, ref_out, atol=1e-2, rtol=1e-2)
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ScaledMM kernel selection logic (CPU-only).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_scaled_mm_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
from abc import ABC
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
_POSSIBLE_KERNELS,
|
||||
_POSSIBLE_NVFP4_KERNELS,
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearKernel,
|
||||
Int8ScaledMMLinearLayerConfig,
|
||||
ScaledMMLinearKernel,
|
||||
_get_linear_backend,
|
||||
_resolve_backend_kernels,
|
||||
init_fp8_linear_kernel,
|
||||
init_int8_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_auto_linear_backend_is_unchanged_without_overrides():
|
||||
kernels: list[type] = [object]
|
||||
quantizations = (
|
||||
"fp8_block_w8a8",
|
||||
"fp8_w8a8",
|
||||
"int8_w8a8",
|
||||
"mixed_precision",
|
||||
"mxfp8",
|
||||
"mxfp4",
|
||||
"mxfp6",
|
||||
"w8a16_fp8",
|
||||
"nvfp4_w4a4",
|
||||
"nvfp4_w4a16",
|
||||
)
|
||||
config = VllmConfig(kernel_config=KernelConfig(linear_backend="auto"))
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
for quantization in quantizations:
|
||||
assert _get_linear_backend(quantization=quantization) == "auto"
|
||||
assert (
|
||||
_resolve_backend_kernels(kernels, "test", quantization=quantization)
|
||||
is kernels
|
||||
)
|
||||
|
||||
|
||||
def test_linear_backend_override_is_quantization_specific():
|
||||
config = VllmConfig(
|
||||
kernel_config=KernelConfig(
|
||||
linear_backend="cutlass",
|
||||
linear_backend_per_quant={"nvfp4_w4a16": "humming"},
|
||||
)
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
assert _get_linear_backend(quantization="nvfp4_w4a16") == "humming"
|
||||
assert _get_linear_backend(quantization="nvfp4_w4a4") == "cutlass"
|
||||
assert _get_linear_backend(quantization="fp8_w8a8") == "cutlass"
|
||||
|
||||
|
||||
@patch.object(HummingFP8ScaledMMLinearKernel, "is_supported", return_value=(True, None))
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_fp8_linear_backend_override(platform_mock, _):
|
||||
platform_mock._enum = PlatformEnum.CUDA
|
||||
config = VllmConfig(
|
||||
kernel_config=KernelConfig(
|
||||
linear_backend="cutlass",
|
||||
linear_backend_per_quant={"fp8_w8a8": "humming"},
|
||||
)
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
kernel = init_fp8_linear_kernel(
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
weight_shape=(128, 128),
|
||||
)
|
||||
|
||||
assert type(kernel) is HummingFP8ScaledMMLinearKernel
|
||||
|
||||
|
||||
def test_is_supported_is_abstract():
|
||||
"""Test that is_supported() is properly defined as abstract."""
|
||||
assert issubclass(ScaledMMLinearKernel, ABC)
|
||||
assert hasattr(ScaledMMLinearKernel, "is_supported")
|
||||
|
||||
|
||||
def test_cpu_kernel_implements_is_supported():
|
||||
"""Test that CPUInt8ScaledMMLinearKernel implements is_supported() method."""
|
||||
assert hasattr(CPUInt8ScaledMMLinearKernel, "is_supported"), (
|
||||
"CPUInt8ScaledMMLinearKernel missing is_supported() method"
|
||||
)
|
||||
# Verify it's a classmethod by checking if it can be called with the class
|
||||
# and by checking the method type
|
||||
assert inspect.ismethod(
|
||||
CPUInt8ScaledMMLinearKernel.is_supported
|
||||
) or inspect.isfunction(CPUInt8ScaledMMLinearKernel.is_supported), (
|
||||
"CPUInt8ScaledMMLinearKernel.is_supported() should be a classmethod"
|
||||
)
|
||||
# Verify it can be called as a classmethod
|
||||
result, reason = CPUInt8ScaledMMLinearKernel.is_supported()
|
||||
assert isinstance(result, bool), "is_supported() should return a bool"
|
||||
assert reason is None or isinstance(reason, str), "reason should be str or None"
|
||||
|
||||
|
||||
def test_aiter_kernel_implements_is_supported():
|
||||
"""Test that AiterInt8ScaledMMLinearKernel implements is_supported() method."""
|
||||
assert hasattr(AiterInt8ScaledMMLinearKernel, "is_supported"), (
|
||||
"AiterInt8ScaledMMLinearKernel missing is_supported() method"
|
||||
)
|
||||
# Verify it's a classmethod by checking if it can be called with the class
|
||||
# and by checking the method type
|
||||
assert inspect.ismethod(
|
||||
AiterInt8ScaledMMLinearKernel.is_supported
|
||||
) or inspect.isfunction(AiterInt8ScaledMMLinearKernel.is_supported), (
|
||||
"AiterInt8ScaledMMLinearKernel.is_supported() should be a classmethod"
|
||||
)
|
||||
# Verify it can be called as a classmethod
|
||||
# (will return False on CPU, which is expected)
|
||||
result, reason = AiterInt8ScaledMMLinearKernel.is_supported()
|
||||
assert isinstance(result, bool), "is_supported() should return a bool"
|
||||
assert reason is None or isinstance(reason, str), "reason should be str or None"
|
||||
# On CPU, it should return False with a reason about requiring ROCm
|
||||
# This validates the method works correctly even on non-ROCm platforms
|
||||
|
||||
|
||||
def test_cpu_kernel_accepts_all_configs():
|
||||
"""Test that CPUInt8ScaledMMLinearKernel accepts all config combinations."""
|
||||
configs = [
|
||||
Int8ScaledMMLinearLayerConfig(
|
||||
is_channelwise=False,
|
||||
is_static_input_scheme=True,
|
||||
input_symmetric=True,
|
||||
),
|
||||
Int8ScaledMMLinearLayerConfig(
|
||||
is_channelwise=True,
|
||||
is_static_input_scheme=False,
|
||||
input_symmetric=False,
|
||||
),
|
||||
]
|
||||
|
||||
for config in configs:
|
||||
can_impl, reason = CPUInt8ScaledMMLinearKernel.can_implement(config)
|
||||
assert can_impl, (
|
||||
f"CPUInt8ScaledMMLinearKernel should accept config {config}: {reason}"
|
||||
)
|
||||
|
||||
|
||||
class OOTInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Int8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_register_oot_linear_kernel(platform_mock):
|
||||
"""Test that the linear kernel registration works correctly."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTInt8ScaledMMLinearKernel, PlatformEnum.OOT, "int8")
|
||||
|
||||
kernel = init_int8_linear_kernel(True, True, True, "module")
|
||||
|
||||
assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), (
|
||||
"init_int8_linear_kernel should return an instance of the registered kernel"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS]
|
||||
)
|
||||
def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels):
|
||||
"""Preferring Humming must not demote Marlin below unrelated kernels."""
|
||||
kernels = possible_kernels[PlatformEnum.CUDA]
|
||||
original = list(kernels)
|
||||
|
||||
assert prioritize_humming(kernels, compute_capability=80) == original
|
||||
|
||||
names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)]
|
||||
humming = next(i for i, name in enumerate(names) if "Humming" in name)
|
||||
assert "Marlin" in names[humming + 1]
|
||||
del names[humming]
|
||||
assert names == [k.__name__ for k in original if "Humming" not in k.__name__]
|
||||
assert kernels == original
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from vllm._custom_ops import scaled_fp4_quant
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip(
|
||||
reason="Nvfp4 Requires compute capability of 10 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
FP4_DTYPE = torch.uint8
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
SHAPES = [(128, 256), (128, 128), (256, 256), (256, 128), (1, 256), (64, 256)]
|
||||
BLOCK_SIZE = 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("shape", SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_silu_mul_nvfp4_quant(
|
||||
default_vllm_config,
|
||||
dtype: torch.dtype,
|
||||
shape: tuple[int, int],
|
||||
) -> None:
|
||||
set_random_seed(42)
|
||||
device = "cuda:0"
|
||||
torch.set_default_device(device)
|
||||
|
||||
x = torch.randn(shape, dtype=dtype)
|
||||
|
||||
# ref op
|
||||
ref_output = SiluAndMul().forward_native(x)
|
||||
ref_global_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.abs(
|
||||
ref_output
|
||||
).max().to(torch.float32)
|
||||
ref_output_quant, ref_block_scale = scaled_fp4_quant(ref_output, ref_global_scale)
|
||||
|
||||
# fused op
|
||||
fused_output_quant = torch.empty_like(ref_output_quant)
|
||||
fused_block_scale = torch.empty_like(ref_block_scale)
|
||||
torch.ops._C.silu_and_mul_nvfp4_quant(
|
||||
fused_output_quant, fused_block_scale, x, ref_global_scale
|
||||
)
|
||||
|
||||
# check dtype
|
||||
assert ref_output_quant.dtype == FP4_DTYPE
|
||||
assert fused_output_quant.dtype == FP4_DTYPE
|
||||
assert ref_output_quant.shape == fused_output_quant.shape
|
||||
|
||||
assert ref_block_scale.dtype == FP8_DTYPE
|
||||
assert fused_block_scale.dtype == FP8_DTYPE
|
||||
assert ref_block_scale.shape == fused_block_scale.shape
|
||||
|
||||
# check dequantized output
|
||||
ref_output_dequant = dequantize_nvfp4_to_dtype(
|
||||
ref_output_quant, ref_block_scale, ref_global_scale, dtype, device
|
||||
)
|
||||
fused_output_dequant = dequantize_nvfp4_to_dtype(
|
||||
fused_output_quant, fused_block_scale, ref_global_scale, dtype, device
|
||||
)
|
||||
|
||||
atol, rtol = 3e-1, 3e-1
|
||||
torch.testing.assert_close(
|
||||
ref_output_dequant, fused_output_dequant, atol=atol, rtol=rtol
|
||||
)
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the triton_scaled_mm kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_triton_scaled_mm.py`.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
device = current_platform.device_type
|
||||
|
||||
triton_scaled_mm_module = importlib.import_module(
|
||||
"vllm.model_executor.layers.quantization.compressed_tensors.triton_scaled_mm"
|
||||
)
|
||||
triton_scaled_mm = triton_scaled_mm_module.triton_scaled_mm
|
||||
|
||||
|
||||
def torch_scaled_mm(
|
||||
a: torch.Tensor,
|
||||
b: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
out_dtype: type[torch.dtype],
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
|
||||
out = scale_a * out
|
||||
out = scale_b.T * out
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
def get_8bit_types():
|
||||
types = [torch.int8]
|
||||
if current_platform.supports_fp8():
|
||||
types.append(current_platform.fp8_dtype())
|
||||
return types
|
||||
|
||||
|
||||
# This test is to check regressions for int8 support on ROCm.
|
||||
@pytest.mark.parametrize(
|
||||
"model_path",
|
||||
[
|
||||
"neuralmagic/Llama-3.2-1B-quantized.w8a8",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Should only run on ROCm")
|
||||
def test_rocm_compressed_tensors_w8a8(
|
||||
vllm_runner, example_prompts, model_path, max_tokens, num_logprobs
|
||||
):
|
||||
dtype = "bfloat16"
|
||||
|
||||
with vllm_runner(model_path, dtype=dtype) as vllm_model:
|
||||
vllm_model.generate_greedy_logprobs(example_prompts, max_tokens, num_logprobs)
|
||||
|
||||
|
||||
MNK_FACTORS = [
|
||||
(1, 256, 128),
|
||||
(33, 256, 496),
|
||||
(64, 971, 1024),
|
||||
(64, 20486, 128),
|
||||
(512, 256, 496),
|
||||
(512, 20486, 1024),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M,N,K", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("in_dtype", get_8bit_types())
|
||||
@pytest.mark.parametrize("use_scalar_scale_a", [True, False])
|
||||
@pytest.mark.parametrize("use_scalar_scale_b", [True, False])
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
def test_scaled_mm(
|
||||
M, N, K, in_dtype, out_dtype, use_scalar_scale_a, use_scalar_scale_b, use_bias
|
||||
):
|
||||
is_floating_point_type = lambda t: torch.tensor([1, 1], dtype=t).is_floating_point()
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
# NOTE: There are cases, where if the matrix is large enough, an output
|
||||
# like 65504.4 can be produced, and can easily turn into inf when
|
||||
# multiplied when using float16/bfloat16. This means one function, e.g.,
|
||||
# testing function, and another function, e.g. golden function, can
|
||||
# produce a non-inf value while the other produces an inf value, and
|
||||
# will cause assert_close/allclose to fail, even though if overflow
|
||||
# wouldn't have occurred, the values would have been "close."
|
||||
#
|
||||
# So, the values here are kept small enough to avoid this situation.
|
||||
if is_floating_point_type(in_dtype):
|
||||
a = (0.25 * torch.rand((M, K), dtype=torch.float32, device=device)).to(in_dtype)
|
||||
b = (0.25 * torch.rand((K, N), dtype=torch.float32, device=device)).to(in_dtype)
|
||||
else:
|
||||
a = torch.randint(-32, 32, (M, K), dtype=in_dtype, device=device)
|
||||
b = torch.randint(-32, 32, (K, N), dtype=in_dtype, device=device)
|
||||
|
||||
if use_scalar_scale_a:
|
||||
scale_a = torch.rand((1, 1), device=device)
|
||||
else:
|
||||
scale_a = 0.25 * torch.rand((M, 1), device=device)
|
||||
|
||||
if use_scalar_scale_b:
|
||||
scale_b = torch.rand((1, 1), device=device)
|
||||
else:
|
||||
scale_b = 0.25 * torch.rand((N, 1), device=device)
|
||||
|
||||
bias = None
|
||||
if use_bias:
|
||||
bias = torch.rand((N,), device=device, dtype=out_dtype)
|
||||
|
||||
c_check = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
c_actual = torch_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias)
|
||||
|
||||
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("in_dtype", get_8bit_types())
|
||||
@pytest.mark.parametrize("use_heuristic", [True, False])
|
||||
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
|
||||
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
|
||||
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
|
||||
a = torch.ones((1, 32), device=device).to(in_dtype)
|
||||
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
|
||||
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
|
||||
scale_a = torch.ones((1, 1), device=device)
|
||||
scale_b = torch.full((1, 1), 2**-10, device=device)
|
||||
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
|
||||
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
|
||||
|
||||
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
|
||||
expected = torch_scaled_mm(*args)
|
||||
assert (expected == 1 + 2**-7).all()
|
||||
torch.testing.assert_close(out, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
# TD operand loads must be bit-exact vs the plain masked-load path.
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Triton scaled_mm runs on CUDA-alike or XPU.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"M,N,K", [(1, 4096, 4096), (64, 4096, 4096), (256, 2048, 4096)]
|
||||
)
|
||||
@pytest.mark.parametrize("in_dtype", get_8bit_types())
|
||||
@pytest.mark.parametrize("use_scalar_scale_a", [True, False])
|
||||
@pytest.mark.parametrize("use_bias", [True, False])
|
||||
def test_scaled_mm_td_matches_plain(M, N, K, in_dtype, use_scalar_scale_a, use_bias):
|
||||
dev = current_platform.device_type
|
||||
out_dtype = torch.bfloat16
|
||||
set_random_seed(0)
|
||||
|
||||
is_fp = torch.tensor([1, 1], dtype=in_dtype).is_floating_point()
|
||||
if is_fp:
|
||||
a = (0.25 * torch.rand((M, K), dtype=torch.float32, device=dev)).to(in_dtype)
|
||||
b = (0.25 * torch.rand((K, N), dtype=torch.float32, device=dev)).to(in_dtype)
|
||||
else:
|
||||
a = torch.randint(-32, 32, (M, K), dtype=in_dtype, device=dev)
|
||||
b = torch.randint(-32, 32, (K, N), dtype=in_dtype, device=dev)
|
||||
|
||||
scale_a = (
|
||||
torch.rand((1, 1), device=dev)
|
||||
if use_scalar_scale_a
|
||||
else 0.25 * torch.rand((M, 1), device=dev)
|
||||
)
|
||||
scale_b = 0.25 * torch.rand((N, 1), device=dev)
|
||||
bias = torch.rand((N,), device=dev, dtype=out_dtype) if use_bias else None
|
||||
|
||||
out_plain = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias, use_td=False)
|
||||
out_td = triton_scaled_mm(a, b, scale_a, scale_b, out_dtype, bias, use_td=True)
|
||||
torch.testing.assert_close(out_td, out_plain, rtol=0, atol=0)
|
||||
@@ -1,490 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ROCm Triton W4A16 GEMM kernel.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_triton_w4a16.py`.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# This test module is ROCm/Triton specific. Avoid import-time failures on
|
||||
# non-ROCm or environments without Triton by skipping early.
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm only", allow_module_level=True)
|
||||
|
||||
pytest.importorskip("triton")
|
||||
|
||||
device = "cuda"
|
||||
|
||||
triton_w4a16_module = importlib.import_module(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.triton_w4a16"
|
||||
)
|
||||
triton_w4a16_gemm = triton_w4a16_module.triton_w4a16_gemm
|
||||
TritonW4A16LinearKernel = triton_w4a16_module.TritonW4A16LinearKernel
|
||||
|
||||
|
||||
def _pack_int4_along_n(w_int4_kn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 values along N: [K, N] -> [K, N//8] int32."""
|
||||
assert w_int4_kn.dtype == torch.int32
|
||||
K, N = w_int4_kn.shape
|
||||
assert N % 8 == 0
|
||||
shifts = torch.arange(8, device=w_int4_kn.device, dtype=torch.int32) * 4
|
||||
return torch.sum(
|
||||
(w_int4_kn.view(K, N // 8, 8) & 0xF) << shifts,
|
||||
dim=2,
|
||||
dtype=torch.int32,
|
||||
).contiguous()
|
||||
|
||||
|
||||
def _unpack_int4_along_n(w_packed_kn8: torch.Tensor) -> torch.Tensor:
|
||||
"""Unpack int4 values along N: [K, N//8] -> [K, N] int32."""
|
||||
assert w_packed_kn8.dtype == torch.int32
|
||||
K, N8 = w_packed_kn8.shape
|
||||
shifts = torch.arange(8, device=w_packed_kn8.device, dtype=torch.int32) * 4
|
||||
nibbles = (w_packed_kn8.unsqueeze(-1) >> shifts) & 0xF
|
||||
return nibbles.reshape(K, N8 * 8)
|
||||
|
||||
|
||||
def _pack_int4_along_k_to_ckpt(w_int4_kn: torch.Tensor) -> torch.Tensor:
|
||||
"""Pack int4 values along K into CT checkpoint layout: [K,N] -> [N, K//8]."""
|
||||
assert w_int4_kn.dtype == torch.int32
|
||||
K, N = w_int4_kn.shape
|
||||
assert K % 8 == 0
|
||||
out = torch.zeros((N, K // 8), dtype=torch.int32, device=w_int4_kn.device)
|
||||
for i in range(8):
|
||||
out |= (w_int4_kn[i::8, :].t() & 0xF) << (i * 4)
|
||||
return out.contiguous()
|
||||
|
||||
|
||||
def _w4a16_reference(
|
||||
a_mk: torch.Tensor,
|
||||
b_packed_kn8: torch.Tensor,
|
||||
scales_gn: torch.Tensor,
|
||||
*,
|
||||
group_size: int,
|
||||
qzeros_gn8: torch.Tensor | None,
|
||||
zp_bias: int,
|
||||
) -> torch.Tensor:
|
||||
"""Reference implementation for W4A16.
|
||||
|
||||
a_mk: [M,K] fp16/bf16
|
||||
b_packed_kn8: [K, N//8] int32, N-packed int4 weights
|
||||
scales_gn: [K//G, N] fp16/bf16
|
||||
qzeros_gn8: [K//G, N//8] int32, N-packed int4 zeros, or None
|
||||
"""
|
||||
assert a_mk.dtype in (torch.float16, torch.bfloat16)
|
||||
assert b_packed_kn8.dtype == torch.int32
|
||||
assert scales_gn.dtype == a_mk.dtype
|
||||
|
||||
M, K = a_mk.shape
|
||||
N = b_packed_kn8.shape[1] * 8
|
||||
assert b_packed_kn8.shape[0] == K
|
||||
|
||||
assert group_size > 0 and K % group_size == 0
|
||||
G = group_size
|
||||
num_groups = K // G
|
||||
assert scales_gn.shape == (num_groups, N)
|
||||
|
||||
w_int4 = _unpack_int4_along_n(b_packed_kn8) # [K,N]
|
||||
if qzeros_gn8 is None:
|
||||
z_full = torch.full((K, N), zp_bias, dtype=torch.int32, device=a_mk.device)
|
||||
else:
|
||||
assert qzeros_gn8.shape == (num_groups, N // 8)
|
||||
z_gn = _unpack_int4_along_n(qzeros_gn8) # [G,N] in groups
|
||||
z_full = z_gn.repeat_interleave(G, dim=0) # [K,N]
|
||||
|
||||
s_full = scales_gn.repeat_interleave(G, dim=0).to(torch.float32) # [K,N]
|
||||
w_fp = (w_int4 - z_full).to(torch.float32) * s_full # [K,N]
|
||||
|
||||
out = a_mk.to(torch.float32) @ w_fp # [M,N]
|
||||
return out.to(a_mk.dtype)
|
||||
|
||||
|
||||
def test_triton_w4a16_gemm_fake_impl():
|
||||
with FakeTensorMode():
|
||||
a = torch.empty((4, 256), device=device, dtype=torch.float16)
|
||||
b_q = torch.empty((256, 64), device=device, dtype=torch.int32)
|
||||
scales = torch.empty((2, 512), device=device, dtype=torch.float16)
|
||||
|
||||
out = triton_w4a16_gemm(
|
||||
a=a,
|
||||
b_q=b_q,
|
||||
scales=scales,
|
||||
qzeros=None,
|
||||
group_size=128,
|
||||
)
|
||||
|
||||
assert out.shape == (4, 512)
|
||||
assert out.dtype == a.dtype
|
||||
assert out.device == a.device
|
||||
|
||||
graph = make_fx(
|
||||
lambda a, b_q, scales: triton_w4a16_gemm(
|
||||
a, b_q, scales, qzeros=None, group_size=128
|
||||
)
|
||||
)(a, b_q, scales)
|
||||
|
||||
assert any(
|
||||
node.op == "call_function"
|
||||
and node.target == torch.ops.vllm.triton_w4a16_gemm.default
|
||||
for node in graph.graph.nodes
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G,has_zp",
|
||||
[
|
||||
(1, 256, 256, 32, False),
|
||||
(17, 256, 512, 32, False),
|
||||
(32, 512, 256, 64, False),
|
||||
(33, 512, 512, 128, False),
|
||||
(64, 1024, 256, 256, False),
|
||||
(128, 256, 1024, 32, True),
|
||||
(64, 512, 512, 64, True),
|
||||
],
|
||||
)
|
||||
def test_triton_w4a16_gemm_matches_reference(dtype, M, K, N, G, has_zp):
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
if N % 8 != 0 or K % G != 0:
|
||||
pytest.skip("Invalid test shape")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
b_packed = _pack_int4_along_n(w_int4)
|
||||
|
||||
scales = (0.05 * torch.rand((K // G, N), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
|
||||
qzeros = None
|
||||
if has_zp:
|
||||
zeros_int4 = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
qzeros = _pack_int4_along_n(zeros_int4)
|
||||
|
||||
out = triton_w4a16_gemm(
|
||||
a=a,
|
||||
b_q=b_packed,
|
||||
scales=scales,
|
||||
qzeros=qzeros,
|
||||
group_size=G,
|
||||
zp_bias=8,
|
||||
)
|
||||
ref = _w4a16_reference(
|
||||
a,
|
||||
b_packed,
|
||||
scales,
|
||||
group_size=G,
|
||||
qzeros_gn8=qzeros,
|
||||
zp_bias=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_gemm_requires_contiguous_inputs():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
set_random_seed(0)
|
||||
M, K, N, G = 32, 256, 256, 32
|
||||
a = torch.randn((K, M), device=device, dtype=torch.float16).t() # non-contiguous
|
||||
w_int4 = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
b_packed = _pack_int4_along_n(w_int4)
|
||||
scales = torch.rand((K // G, N), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
triton_w4a16_gemm(
|
||||
a=a,
|
||||
b_q=b_packed,
|
||||
scales=scales,
|
||||
qzeros=None,
|
||||
group_size=G,
|
||||
zp_bias=8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_process_weights_after_loading_repacks_layout():
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedColumnParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:0",
|
||||
local_rank=0,
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
# Small-but-nontrivial shapes.
|
||||
K, N = 256, 256
|
||||
G = 32
|
||||
assert K % 8 == 0 and N % 8 == 0 and K % G == 0
|
||||
|
||||
# Build a canonical int4 weight grid then pack into the CT checkpoint layout.
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_ckpt_nk8 = _pack_int4_along_k_to_ckpt(w_int4_kn) # [N, K//8]
|
||||
|
||||
# Scales in CT checkpoint layout for WNA16: [N, K//G]
|
||||
scales_ckpt_nkg = 0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
# Asymmetric case: zero points in CT checkpoint layout [N//8, K//G] (N-packed)
|
||||
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
zeros_packed_gn8 = _pack_int4_along_n(zeros_int4_gn) # [K//G, N//8]
|
||||
zeros_ckpt_n8kg = zeros_packed_gn8.t().contiguous() # [N//8, K//G]
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4, # asymmetric
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name="weight_zero_point",
|
||||
)
|
||||
|
||||
# Build dummy layer with vLLM parameter wrappers.
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"weight_packed",
|
||||
PackedvLLMParameter(
|
||||
data=w_ckpt_nk8,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=1,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_ckpt_nkg,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_zero_point",
|
||||
PackedColumnParameter(
|
||||
data=zeros_ckpt_n8kg,
|
||||
weight_loader=weight_loader,
|
||||
output_dim=0,
|
||||
packed_factor=8,
|
||||
packed_dim=0,
|
||||
),
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
# Expected transformed layouts.
|
||||
expected_w_kn8 = _pack_int4_along_n(w_int4_kn) # [K, N//8]
|
||||
expected_scales_gn = scales_ckpt_nkg.t().contiguous() # [K//G, N]
|
||||
expected_zeros_gn8 = zeros_ckpt_n8kg.t().contiguous() # [K//G, N//8]
|
||||
|
||||
assert tuple(layer.weight_packed.shape) == (K, N // 8)
|
||||
assert tuple(layer.weight_scale.shape) == (K // G, N)
|
||||
assert tuple(layer.weight_zero_point.shape) == (K // G, N // 8)
|
||||
|
||||
torch.testing.assert_close(layer.weight_packed, expected_w_kn8)
|
||||
torch.testing.assert_close(layer.weight_scale, expected_scales_gn)
|
||||
torch.testing.assert_close(layer.weight_zero_point, expected_zeros_gn8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_process_weights_after_loading_keeps_gptq_qzeros_layout():
|
||||
"""AutoGPTQ qzeros are already [K//G, N//8] (output_dim=1): no transpose."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method="tcp://127.0.0.1:0",
|
||||
local_rank=0,
|
||||
)
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
K, N = 256, 256
|
||||
G = 32
|
||||
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
w_gptq_k8n = _pack_int4_along_k_to_ckpt(w_int4_kn).t().contiguous()
|
||||
scales_gptq_gn = 0.05 * torch.rand((K // G, N), device=device, dtype=torch.float16)
|
||||
zeros_int4_gn = torch.randint(0, 16, (K // G, N), device=device, dtype=torch.int32)
|
||||
zeros_gptq_gn8 = _pack_int4_along_n(zeros_int4_gn)
|
||||
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=True,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
)
|
||||
|
||||
weight_loader = lambda *args, **kwargs: None
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.register_parameter(
|
||||
"qweight",
|
||||
PackedvLLMParameter(
|
||||
data=w_gptq_k8n,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_factor=8,
|
||||
packed_dim=0,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"scales",
|
||||
GroupQuantScaleParameter(
|
||||
data=scales_gptq_gn,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"qzeros",
|
||||
PackedvLLMParameter(
|
||||
data=zeros_gptq_gn8,
|
||||
weight_loader=weight_loader,
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_factor=8,
|
||||
packed_dim=1,
|
||||
),
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
expected_w_kn8 = _pack_int4_along_n(w_int4_kn)
|
||||
|
||||
assert tuple(layer.qweight.shape) == (K, N // 8)
|
||||
assert tuple(layer.scales.shape) == (K // G, N)
|
||||
assert tuple(layer.qzeros.shape) == (K // G, N // 8)
|
||||
|
||||
torch.testing.assert_close(layer.qweight, expected_w_kn8)
|
||||
torch.testing.assert_close(layer.scales, scales_gptq_gn)
|
||||
torch.testing.assert_close(layer.qzeros, zeros_gptq_gn8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_triton_w4a16_symmetric_apply_ignores_qzeros(monkeypatch):
|
||||
"""For symmetric (uint4b8) layers, apply_weights must pass qzeros=None."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
K, N, G = 256, 256, 32
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
)
|
||||
kernel = TritonW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="qweight",
|
||||
w_s_param_name="scales",
|
||||
w_zp_param_name="qzeros",
|
||||
)
|
||||
|
||||
class DummyLayer(torch.nn.Module):
|
||||
pass
|
||||
|
||||
layer = DummyLayer()
|
||||
layer.qweight = torch.empty((K, N // 8), device=device, dtype=torch.int32)
|
||||
layer.scales = torch.empty((K // G, N), device=device, dtype=torch.float16)
|
||||
layer.qzeros = torch.empty((1, 1), device=device, dtype=torch.int32)
|
||||
|
||||
captured = {}
|
||||
|
||||
def fake_gemm(*, a, b_q, scales, qzeros, group_size, zp_bias):
|
||||
captured["qzeros"] = qzeros
|
||||
captured["zp_bias"] = zp_bias
|
||||
return torch.empty((a.shape[0], N), device=a.device, dtype=a.dtype)
|
||||
|
||||
monkeypatch.setattr(triton_w4a16_module, "triton_w4a16_gemm", fake_gemm)
|
||||
|
||||
x = torch.empty((2, K), device=device, dtype=torch.float16)
|
||||
output = kernel.apply_weights(layer, x)
|
||||
|
||||
assert output.shape == (2, N)
|
||||
assert captured["qzeros"] is None
|
||||
assert captured["zp_bias"] == scalar_types.uint4b8.bias
|
||||
@@ -1,70 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for W4A16 kernel selection logic (ROCm).
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_w4a16_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MPLinearLayerConfig,
|
||||
choose_mp_linear_kernel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx1100
|
||||
else:
|
||||
on_gfx1100 = on_gfx1x = lambda: False # noqa: E731
|
||||
|
||||
# Group sizes the HIP skinny path of RDNAHybridW4A16LinearKernel instantiates.
|
||||
_HYBRID_GROUP_SIZES = (32, 64, 128)
|
||||
|
||||
|
||||
def _expected_rocm_kernel(weight_type, group_size: int) -> str:
|
||||
"""Mirror the ROCm priority order in ``_POSSIBLE_KERNELS[ROCM]``:
|
||||
RDNA3 (gfx1100, symmetric uint4b8) -> Hybrid (gfx11/gfx12) -> Triton.
|
||||
"""
|
||||
if on_gfx1100() and weight_type == scalar_types.uint4b8:
|
||||
return "RDNA3W4A16LinearKernel"
|
||||
if on_gfx1x() and group_size in _HYBRID_GROUP_SIZES:
|
||||
return "RDNAHybridW4A16LinearKernel"
|
||||
return "TritonW4A16LinearKernel"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_choose_mp_linear_kernel_uint4b8():
|
||||
# int4 weights, 16-bit activations (CT W4A16 typical config).
|
||||
K, N = 1024, 256
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8, # symmetric int4 (bias=8)
|
||||
act_type=torch.float16,
|
||||
group_size=128,
|
||||
zero_points=False,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
assert kernel_type.__name__ == _expected_rocm_kernel(scalar_types.uint4b8, 128)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_choose_mp_linear_kernel_uint4_asymmetric():
|
||||
# Asymmetric int4 weights (explicit zero points).
|
||||
K, N = 512, 512
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4, # asymmetric int4 (explicit zeros)
|
||||
act_type=torch.bfloat16,
|
||||
group_size=64,
|
||||
zero_points=True,
|
||||
)
|
||||
|
||||
kernel_type = choose_mp_linear_kernel(config)
|
||||
assert kernel_type.__name__ == _expected_rocm_kernel(scalar_types.uint4, 64)
|
||||
@@ -1,661 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
|
||||
|
||||
The zentorch ops are mocked with reference implementations when zentorch is not
|
||||
installed, so the layout and dispatch contracts are covered in CI.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
|
||||
|
||||
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
_import_unpack_from_int32,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
GROUP_SIZE = 128
|
||||
IN_FEATURES = 512
|
||||
OUT_FEATURES = 128
|
||||
|
||||
|
||||
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
|
||||
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
|
||||
words = packed.view(torch.int32)
|
||||
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
|
||||
for i in range(8):
|
||||
nibble = (words >> (4 * i)) & 0xF
|
||||
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
|
||||
return out
|
||||
|
||||
|
||||
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
|
||||
n, k = unpacked.shape
|
||||
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
|
||||
out = torch.zeros(n, k // 8, dtype=torch.int32)
|
||||
for i in range(8):
|
||||
out |= (values[:, :, i] & 0xF) << (4 * i)
|
||||
return out
|
||||
|
||||
|
||||
def _quantize_per_group(
|
||||
weight: torch.Tensor, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
|
||||
out_features, in_features = weight.shape
|
||||
grouped = weight.float().reshape(
|
||||
out_features, in_features // group_size, group_size
|
||||
)
|
||||
scale = grouped.abs().amax(dim=-1) / 8.0
|
||||
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
|
||||
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _dequantize_per_group(
|
||||
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
|
||||
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
|
||||
|
||||
|
||||
def _dynamic_quant_matmul(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
|
||||
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
|
||||
out = (x_q.float() @ weight.t()) * scale
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_zentorch_ops():
|
||||
"""Register reference zentorch ops when zentorch is not installed."""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
|
||||
yield None
|
||||
return
|
||||
|
||||
calls: dict[str, tuple] = {}
|
||||
|
||||
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
|
||||
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
|
||||
in_features = weight.view(torch.int32).shape[1] * 8
|
||||
group_size = in_features // weight_scales.shape[0]
|
||||
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
|
||||
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
|
||||
weight_deq = _unpack_s4(weight, in_features) * scales
|
||||
out = _dynamic_quant_matmul(input, weight_deq, bias)
|
||||
return out.to(input.dtype)
|
||||
|
||||
def _fused_moe(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales=None,
|
||||
w2_scales=None,
|
||||
zentorch_op_name="",
|
||||
):
|
||||
calls["fused_moe"] = (
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
)
|
||||
output.zero_()
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
|
||||
lib_def.define(
|
||||
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
|
||||
"Tensor weight_scales, Tensor? bias=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
|
||||
)
|
||||
lib_def.define(
|
||||
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
|
||||
"Tensor topk_id, bool skip_weighted, str act, "
|
||||
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
|
||||
)
|
||||
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
|
||||
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
|
||||
lib_impl.impl("zentorch_fused_moe", _fused_moe)
|
||||
|
||||
yield calls
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
def _make_config(
|
||||
act_type: torch.dtype = torch.bfloat16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
zero_points: bool = False,
|
||||
group_size: int = GROUP_SIZE,
|
||||
) -> MPLinearLayerConfig:
|
||||
return MPLinearLayerConfig(
|
||||
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
weight_type=weight_type,
|
||||
act_type=act_type,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
|
||||
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
|
||||
quantized, scale = _quantize_per_group(weight, group_size)
|
||||
|
||||
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
|
||||
packed = pack_to_int32(quantized, 4, packed_dim=1)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
|
||||
layer.weight_packed.input_dim = 1
|
||||
layer.weight_packed.packed_dim = 1
|
||||
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
|
||||
layer.weight_zero_point = None
|
||||
return layer, _dequantize_per_group(quantized, scale, group_size)
|
||||
|
||||
|
||||
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
|
||||
return ZentorchWNA16LinearKernel(
|
||||
config, "weight_packed", "weight_scale", "weight_zero_point"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dense DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config_kwargs,reason",
|
||||
[
|
||||
({"act_type": torch.float32}, "f32 activations are rejected"),
|
||||
(
|
||||
{"weight_type": scalar_types.uint4, "zero_points": True},
|
||||
"asymmetric int4 is unsupported",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config(**config_kwargs))
|
||||
assert not kernel._maybe_process_da8w4_weights(layer), reason
|
||||
|
||||
|
||||
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"envs.VLLM_CPU_INT4_W4A8",
|
||||
False,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._zentorch_da8w4
|
||||
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
|
||||
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
|
||||
assert layer._zentorch_da8w4_packed.dtype == torch.int8
|
||||
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
|
||||
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
|
||||
assert layer._zentorch_da8w4_scale.shape == (
|
||||
IN_FEATURES // GROUP_SIZE,
|
||||
OUT_FEATURES,
|
||||
)
|
||||
assert layer._zentorch_da8w4_packed.is_contiguous()
|
||||
assert layer._zentorch_da8w4_scale.is_contiguous()
|
||||
# The checkpoint parameters are released once repacked.
|
||||
assert layer.weight_packed.numel() == 0
|
||||
assert layer.weight_scale.numel() == 0
|
||||
|
||||
|
||||
def test_da8w4_apply_weights_matches_dequantized_reference(
|
||||
mock_zentorch_ops, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, weight_deq = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
|
||||
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
|
||||
out = kernel.apply_weights(layer, x, bias)
|
||||
|
||||
expected = _dynamic_quant_matmul(x, weight_deq, bias)
|
||||
assert out.dtype == torch.bfloat16
|
||||
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
|
||||
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: "zentorch_dynamic_qlinear" not in ops,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_can_implement_requires_zen_cpu(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: False,
|
||||
)
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
|
||||
assert not ok
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
|
||||
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
|
||||
CPUWNA16LinearKernel,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
|
||||
"current_platform.is_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
unaligned = dataclasses.replace(
|
||||
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
|
||||
)
|
||||
assert unaligned.partition_weight_shape[1] % 32 != 0
|
||||
|
||||
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
|
||||
assert not ok, "the parent is expected to turn this shape away"
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
|
||||
assert ok, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[
|
||||
(-1, True), # per-channel, one group spanning K
|
||||
(GROUP_SIZE, True),
|
||||
(IN_FEATURES, True),
|
||||
(6, False), # even, so the parent's multiple-of-2 rule would allow it
|
||||
(0, False),
|
||||
],
|
||||
)
|
||||
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
|
||||
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(
|
||||
_make_config(group_size=group_size)
|
||||
)
|
||||
assert ok is expected_ok, reason
|
||||
if not expected_ok:
|
||||
assert reason is not None
|
||||
assert "must divide input size" in reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MoE DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NUM_EXPERTS = 4
|
||||
HIDDEN = 256
|
||||
INTERMEDIATE = 128
|
||||
|
||||
|
||||
def _make_moe_weights(group_size: int = 32):
|
||||
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
|
||||
torch.manual_seed(0)
|
||||
w13_q, w13_s, w2_q, w2_s = [], [], [], []
|
||||
for _ in range(NUM_EXPERTS):
|
||||
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
|
||||
q13, s13 = _quantize_per_group(w13, group_size)
|
||||
# CT is N-first and packs along the input dim: [N, K//8] per expert.
|
||||
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
|
||||
w13_s.append(s13)
|
||||
|
||||
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
|
||||
q2, s2 = _quantize_per_group(w2, group_size)
|
||||
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
|
||||
w2_s.append(s2)
|
||||
return (
|
||||
torch.stack(w13_q),
|
||||
torch.stack(w2_q),
|
||||
torch.stack(w13_s),
|
||||
torch.stack(w2_s),
|
||||
)
|
||||
|
||||
|
||||
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
|
||||
backends = int_wna16._get_priority_backends()
|
||||
assert backends[0] == WNA16MoEBackend.ZEN_CPU
|
||||
# The generic CPU backend stays available as a fallback.
|
||||
assert WNA16MoEBackend.CPU in backends
|
||||
|
||||
|
||||
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
|
||||
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
|
||||
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
|
||||
|
||||
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
|
||||
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
assert w13_out.dtype == w2_out.dtype == torch.int32
|
||||
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
|
||||
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
|
||||
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
|
||||
# Symmetric checkpoints carry no zero points into the kernel.
|
||||
assert converted[8] is None and converted[9] is None
|
||||
|
||||
|
||||
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
|
||||
"""The repacked weight must dequantize back to the checkpoint values."""
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
|
||||
expected = _import_unpack_from_int32()(
|
||||
w13,
|
||||
4,
|
||||
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
|
||||
packed_dim=1,
|
||||
)
|
||||
for expert in range(NUM_EXPERTS):
|
||||
torch.testing.assert_close(
|
||||
_unpack_s4(w13_out[expert], HIDDEN),
|
||||
expected[expert].float(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[(32, True), (128, True), (-1, False), (2, False)],
|
||||
)
|
||||
def test_zen_cpu_group_size_gating(group_size, expected_ok):
|
||||
quant_config = type("Args", (), {"group_size": group_size})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"may_have_zp,may_have_bias,expected_ok",
|
||||
[(True, False, False), (False, True, True), (True, True, False)],
|
||||
)
|
||||
def test_zen_cpu_rejects_zero_points_but_takes_bias(
|
||||
may_have_zp, may_have_bias, expected_ok
|
||||
):
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=may_have_zp,
|
||||
may_have_bias=may_have_bias,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
def test_zen_cpu_rejects_moe_wna16_layout():
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
quant_config = MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
has_zp=False,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=None,
|
||||
full_config={},
|
||||
)
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_disabled_by_env(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_backend_maps_to_experts_class():
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
|
||||
ZentorchExpertsInt4
|
||||
]
|
||||
|
||||
|
||||
def test_zen_experts_support_predicates():
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
|
||||
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
|
||||
for act in MoEActivation:
|
||||
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
|
||||
assert ZentorchExpertsInt4._supports_activation(act) == expected
|
||||
assert ZentorchExpertsInt4.requires_interleaved_w13
|
||||
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
|
||||
assert ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": False})()
|
||||
)
|
||||
assert not ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": True})()
|
||||
)
|
||||
|
||||
|
||||
def test_zen_experts_take_custom_routing():
|
||||
"""Models with their own router (gemma-4) reach select_experts through the
|
||||
config captured off the layer, since apply() cannot be handed the callable."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_routing_method(
|
||||
RoutingMethodType.Custom, None, None
|
||||
)
|
||||
|
||||
def routing_fn(**kwargs):
|
||||
raise AssertionError("not called")
|
||||
|
||||
layer = _StubMoELayer(
|
||||
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
|
||||
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
|
||||
MoEActivation.SILU,
|
||||
)
|
||||
layer.renormalize = True
|
||||
layer.custom_routing_function = routing_fn
|
||||
|
||||
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
|
||||
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
|
||||
|
||||
assert experts.custom_routing_function is routing_fn
|
||||
assert experts.renormalize is True
|
||||
|
||||
|
||||
def _swigluoai_perm(activation, experts_cls=None):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
swigluoai_w13_interleave_perm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
return swigluoai_w13_interleave_perm(
|
||||
experts_cls or ZentorchExpertsInt4,
|
||||
activation,
|
||||
2 * INTERMEDIATE,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def test_swigluoai_perm_only_for_zen_swigluoai():
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
|
||||
# Other activations keep the half-split layout the loader leaves.
|
||||
assert _swigluoai_perm(MoEActivation.SILU) is None
|
||||
# object stands in for any kernel that never set requires_interleaved_w13.
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
|
||||
|
||||
|
||||
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
|
||||
"""The permutation is a gather on w13's output-channel axis, which is dim 1
|
||||
for the N-first packed weights and their group scales."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
|
||||
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
|
||||
|
||||
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
permuted = int_wna16._process_weights_zen_cpu(
|
||||
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
|
||||
)[0]
|
||||
|
||||
for expert in range(NUM_EXPERTS):
|
||||
before = _unpack_s4(baseline[expert], HIDDEN)
|
||||
after = _unpack_s4(permuted[expert], HIDDEN)
|
||||
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
|
||||
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
|
||||
|
||||
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
|
||||
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])
|
||||
Reference in New Issue
Block a user