[ROCm][CI] Move ROCm AITER quantization tests (#40938)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andreas Karatzas
2026-08-18 14:50:35 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent d785eb51ce
commit 5fa8ca971a
7 changed files with 144 additions and 136 deletions
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ROCm AITER MLA FP8 support detection."""
import sys
import types
from typing import Any
from unittest.mock import patch
import pytest
from vllm.platforms import current_platform
_SKIP_UNSUPPORTED_AITER_HARDWARE = True
if current_platform.is_rocm():
from vllm.platforms.rocm import get_cdna_version
_SKIP_UNSUPPORTED_AITER_HARDWARE = get_cdna_version() <= 2
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.fixture(autouse=True)
def reset_aiter_mla_support_cache(monkeypatch: pytest.MonkeyPatch) -> None:
import vllm._aiter_ops as aiter_ops
monkeypatch.setattr(aiter_ops, "_AITER_MLA_SUPPORTS_FP8", None)
def _install_fake_aiter_modules(
monkeypatch: pytest.MonkeyPatch, *, supports_fp8: bool
) -> None:
aiter_mod: Any = types.ModuleType("aiter")
mla_mod: Any = types.ModuleType("aiter.mla")
if supports_fp8:
def mla_decode_fwd_with_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
q_scale=None,
kv_scale=None,
):
return None
mla_decode_fwd: Any = mla_decode_fwd_with_fp8
else:
def mla_decode_fwd_without_fp8(
q,
kv_buffer,
kv_indptr,
kv_indices,
o,
sm_scale,
):
return None
mla_decode_fwd = mla_decode_fwd_without_fp8
mla_mod.mla_decode_fwd = mla_decode_fwd
aiter_mod.mla = mla_mod
monkeypatch.setitem(sys.modules, "aiter", aiter_mod)
monkeypatch.setitem(sys.modules, "aiter.mla", mla_mod)
def test_aiter_mla_fp8_support_detects_fp8_signature(monkeypatch):
"""The support check should detect q_scale and kv_scale parameters."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
assert _check_aiter_mla_fp8_support() is True
def test_aiter_mla_fp8_support_rejects_missing_fp8_signature(monkeypatch):
"""The support check should return False when FP8 params are absent."""
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=False)
assert _check_aiter_mla_fp8_support() is False
@pytest.mark.skipif(
_SKIP_UNSUPPORTED_AITER_HARDWARE,
reason="Installed AITER MLA FP8 check requires CDNA 3 or newer",
)
def test_installed_aiter_mla_supports_fp8():
"""Supported ROCm CI must provide AITER with MLA FP8 scaling."""
from vllm._aiter_ops import (
_check_aiter_mla_fp8_support,
is_aiter_found_and_supported,
)
assert is_aiter_found_and_supported(), (
"AITER must be installed on supported ROCm hardware"
)
assert _check_aiter_mla_fp8_support() is True
@pytest.mark.parametrize(
"error_type",
[ImportError, ModuleNotFoundError, AttributeError, ValueError, TypeError],
)
def test_aiter_mla_fp8_support_handles_signature_errors(monkeypatch, error_type):
"""The support check should fail closed on import or signature problems."""
import vllm._aiter_ops as aiter_ops
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", side_effect=error_type("boom")):
assert _check_aiter_mla_fp8_support() is False
assert aiter_ops._AITER_MLA_SUPPORTS_FP8 is False
def test_aiter_mla_fp8_support_result_is_cached(monkeypatch):
"""The support check should reuse the cached result on later calls."""
import inspect
from vllm._aiter_ops import _check_aiter_mla_fp8_support
_install_fake_aiter_modules(monkeypatch, supports_fp8=True)
with patch("inspect.signature", wraps=inspect.signature) as signature_mock:
assert _check_aiter_mla_fp8_support() is True
assert _check_aiter_mla_fp8_support() is True
assert signature_mock.call_count == 1
@@ -0,0 +1,407 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import csv
import importlib
import importlib.util
import os
import pytest
import torch
from tests.utils import TestFP8Layer
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.scaled_mm.aiter import (
AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.ScaledMMLinearKernel import (
FP8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
get_fp8_min_max,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = [
pytest.mark.skipif(
not (
current_platform.is_rocm()
and current_platform.supports_fp8()
and aiter_available
),
reason="Requires ROCm + FP8 support + aiter",
),
pytest.mark.usefixtures("default_vllm_config"),
]
@pytest.fixture
def enable_hipb_mm_kernel(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
rocm_aiter_ops.refresh_env_variables()
yield
rocm_aiter_ops.refresh_env_variables()
def _make_config(
*,
weight_quant_key=kFp8StaticChannelSym,
out_dtype: torch.dtype = torch.bfloat16,
weight_shape: tuple[int, int] = (512, 4096),
) -> FP8ScaledMMLinearLayerConfig:
return FP8ScaledMMLinearLayerConfig(
weight_quant_key=weight_quant_key,
activation_quant_key=kFp8DynamicTokenSym,
weight_shape=weight_shape,
input_dtype=torch.bfloat16,
out_dtype=out_dtype,
)
def _find_csv_row(path: str, m: int, n: int, k: int) -> dict | None:
if not os.path.exists(path):
return None
with open(path, newline="") as f:
reader = csv.DictReader(f, skipinitialspace=True)
for row in reader:
try:
if (
int(row.get("m", -1)) == m
and int(row.get("n", -1)) == n
and int(row.get("k", -1)) == k
):
return dict(row)
except (TypeError, ValueError):
continue
return None
def _skip_if_no_hipb_mm_solution(exc: RuntimeError) -> None:
if "hipblasLtMatmulAlgoGetHeuristic found 0 valid solutions" in str(exc):
pytest.skip(
"hipb_mm bpreshuffle path has no valid hipBLASLt solution on "
"this ROCm stack."
)
def _check_bpreshuffle_runtime_support(weight_shape: tuple[int, int], num_tokens: int):
import aiter
from aiter.ops.shuffle import shuffle_weight
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device="cuda")
w = torch.randn(weight_shape, dtype=torch.bfloat16, device="cuda")
aiter.hipb_create_extension()
x_q, x_scale = aiter.pertoken_quant(x, quant_dtype=current_platform.fp8_dtype())
w_q, w_scale = aiter.pertoken_quant(w, quant_dtype=current_platform.fp8_dtype())
try:
aiter.hipb_mm(
x_q,
shuffle_weight(w_q, layout=(16, 16)).t(),
solution_index=-1,
out_dtype=torch.bfloat16,
scaleA=x_scale,
scaleB=w_scale.t().contiguous(),
scaleOut=None,
bpreshuffle=True,
)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
def test_hipb_mm_kernel_requires_hipbmm_flag(monkeypatch: pytest.MonkeyPatch):
# The kernel rejects when `is_hip_fp8bmm_enabled()` is False. That helper
# requires AITER + AITER_LINEAR + MI3xx, so dropping AITER_LINEAR exercises
# the rejection branch.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "0")
monkeypatch.delenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", raising=False)
rocm_aiter_ops.refresh_env_variables()
is_supported, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.is_supported()
assert not is_supported
assert reason == (
"requires setting `VLLM_ROCM_USE_AITER=1`, "
"`VLLM_ROCM_USE_AITER_LINEAR=1`, "
"and `VLLM_ROCM_USE_AITER_LINEAR_HIPBMM=1`."
)
def test_hipb_mm_flag_enables_hip_online_tuning(
monkeypatch: pytest.MonkeyPatch,
):
import vllm.envs as envs_mod
import vllm.platforms.rocm as rocm_mod
# The rocm.py gate requires all three AITER flags (and MI3xx) to auto-set
# HIP_ONLINE_TUNING.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR_HIPBMM", "1")
try:
importlib.reload(envs_mod)
importlib.reload(rocm_mod)
assert envs_mod.VLLM_ROCM_USE_AITER
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR
assert envs_mod.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
assert os.environ.get("HIP_ONLINE_TUNING") == "1"
finally:
monkeypatch.undo()
os.environ.pop("HIP_ONLINE_TUNING", None)
importlib.reload(envs_mod)
importlib.reload(rocm_mod)
rocm_aiter_ops.refresh_env_variables()
def test_hipb_mm_kernel_can_implement_success(enable_hipb_mm_kernel):
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
_make_config()
)
assert can_implement
assert reason is None
@pytest.mark.parametrize(
("config", "expected_reason"),
[
(
_make_config(weight_quant_key=kFp8StaticTensorSym),
"requires per token activation scales and per channel weight scales.",
),
(
_make_config(out_dtype=torch.float16),
"requires bfloat16 output dtype.",
),
(
_make_config(weight_shape=(8, 4090)),
"requires N >= 16 and both N and K divisible by 16, "
"received N=8 and K=4090.",
),
],
)
def test_hipb_mm_kernel_can_implement_rejects_unsupported_configs(
enable_hipb_mm_kernel,
config: FP8ScaledMMLinearLayerConfig,
expected_reason: str,
):
can_implement, reason = AiterHipbMMPerTokenFp8ScaledMMLinearKernel.can_implement(
config
)
assert not can_implement
assert reason == expected_reason
def test_hipb_mm_kernel_process_weights_after_loading_shuffles_weights(
enable_hipb_mm_kernel,
):
weight_shape = (512, 4096)
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
_make_config(weight_shape=weight_shape),
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.rand(weight_shape, device="cuda").to(current_platform.fp8_dtype()).t(),
requires_grad=False,
)
layer.weight_scale = torch.nn.Parameter(
torch.rand((weight_shape[0], 1), dtype=torch.float32, device="cuda"),
requires_grad=False,
)
layer.input_scale = None
layer.input_scale_ub = None
original_weight = layer.weight.detach().clone()
original_weight_scale = layer.weight_scale.detach().clone()
kernel.process_weights_after_loading(layer)
# process_weights_after_loading now pre-applies the transposes that used
# to live in _rocm_aiter_hipb_mm_fp8_impl, so the stored weight is the
# shuffled tensor with a trailing `.t()` view, and the stored weight scale
# is its transposed-contiguous form.
expected_weight = rocm_aiter_ops.shuffle_weight(
original_weight.t().contiguous()
).t()
torch.testing.assert_close(layer.weight, expected_weight)
expected_weight_scale = original_weight_scale.t().contiguous()
torch.testing.assert_close(layer.weight_scale, expected_weight_scale)
def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel):
import aiter
weight_shape = (512, 4096)
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=32)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticChannelSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
# hipb_mm uses a transposed-result GEMM internally, so the flattened token
# count becomes the effective N dimension passed into hipBLASLt. Keep it
# aligned to avoid heuristic failures for tiny N.
x = torch.randn(2, 16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device="cuda")
try:
out = layer(x, bias)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
x_2d = x.view(-1, x.shape[-1])
x_q, x_scale = layer.kernel.quant_fp8(
x_2d,
layer.input_scale,
layer.input_scale_ub,
)
try:
# process_weights_after_loading already applies the trailing `.t()` on
# the shuffled weight and the `.t().contiguous()` on the weight scale,
# so the raw aiter call uses them directly.
expected = aiter.hipb_mm(
x_q,
layer.weight,
solution_index=-1,
bias=bias,
out_dtype=torch.bfloat16,
scaleA=x_scale,
scaleB=layer.weight_scale,
scaleOut=None,
bpreshuffle=True,
).view(*out.shape)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
assert isinstance(layer.kernel, AiterHipbMMPerTokenFp8ScaledMMLinearKernel)
assert out.shape == (2, 16, weight_shape[0])
torch.testing.assert_close(out, expected)
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
"""Kernel output should match a dequantized fp32 reference within
fp8 per-token / per-channel quantization noise."""
weight_shape = (512, 4096) # (N, K)
num_tokens = 32
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
fp8_dtype = current_platform.fp8_dtype()
fp8_max = get_fp8_min_max()[1]
device = torch.device("cuda")
# Build a bf16 weight and quantize per output channel (one scale per row).
w_bf16 = torch.randn(weight_shape, dtype=torch.bfloat16, device=device)
w_amax = w_bf16.abs().amax(dim=1, keepdim=True).to(torch.float32)
w_scale = (w_amax / fp8_max).clamp(min=1e-12)
w_fp8 = (w_bf16.to(torch.float32) / w_scale).clamp(-fp8_max, fp8_max).to(fp8_dtype)
w_dequant = w_fp8.to(torch.float32) * w_scale
bias = torch.randn(weight_shape[0], dtype=torch.bfloat16, device=device)
layer = torch.nn.Module()
# Pre-`process_weights_after_loading` convention: weight stored as the
# `[K, N]` view of the fp8 tensor.
layer.weight = torch.nn.Parameter(w_fp8.t(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale, requires_grad=False)
layer.input_scale = None
layer.input_scale_ub = None
kernel = AiterHipbMMPerTokenFp8ScaledMMLinearKernel(
_make_config(weight_shape=weight_shape),
layer_param_names=("weight", "weight_scale", "input_scale", "input_scale_ub"),
)
kernel.process_weights_after_loading(layer)
x = torch.randn(num_tokens, weight_shape[1], dtype=torch.bfloat16, device=device)
try:
out = kernel.apply_weights(layer, x, bias)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
# Reference: quantize x per-token the same way the kernel does, then run
# the matmul in fp32 against the dequantized weight. This isolates plumbing
# / reduction bugs from inherent fp8 quantization noise.
x_amax = x.abs().amax(dim=1, keepdim=True).to(torch.float32)
x_scale_ref = (x_amax / fp8_max).clamp(min=1e-12)
x_q = (x.to(torch.float32) / x_scale_ref).clamp(-fp8_max, fp8_max).to(fp8_dtype)
x_dequant = x_q.to(torch.float32) * x_scale_ref
expected = (x_dequant @ w_dequant.t() + bias.to(torch.float32)).to(torch.bfloat16)
assert out.shape == (num_tokens, weight_shape[0])
# K=4096 fp8 reduction leaves room for accumulation order drift and
# catastrophic cancellation on near-zero outputs; tolerances are loose
# enough to absorb that but tight enough to catch wrong layouts, missing
# bias, swapped scales, etc.
torch.testing.assert_close(out, expected, atol=5.0, rtol=0.1)
def test_hipb_mm_kernel_online_tuning_writes_csv(
enable_hipb_mm_kernel,
monkeypatch: pytest.MonkeyPatch,
tmp_path,
):
weight_shape = (256, 4096)
cache_file = tmp_path / "hip_online_tuning_res.csv"
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=16)
monkeypatch.setenv("HIP_ONLINE_TUNING", "1")
monkeypatch.chdir(tmp_path)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticChannelSym,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=AiterHipbMMPerTokenFp8ScaledMMLinearKernel,
)
# The effective heuristic N dimension is the flattened token count.
x = torch.randn(16, weight_shape[1], dtype=torch.bfloat16, device="cuda")
try:
out = layer(x)
except RuntimeError as exc:
_skip_if_no_hipb_mm_solution(exc)
raise
torch.accelerator.synchronize()
assert out.shape == (16, weight_shape[0])
assert cache_file.exists()
# hipb_mm records the internal GEMM dimensions used by hipBLASLt after its
# transposed-result transformation.
row = _find_csv_row(
str(cache_file),
m=weight_shape[0],
n=x.shape[0],
k=weight_shape[1],
)
assert row is not None
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Schema/aliasing tests for the AITER FP8 quantization custom ops.
#
# These use the shared opcheck helper, whose test_schema check catches custom
# ops whose implementation aliases an input that the registered schema declares
# as non-aliasing -- the failure mode behind the rocm_aiter_per_tensor_quant
# regression (a returned scale that aliased the input scale).
#
# Skipped if AITER is not installed or the platform is not ROCm.
import importlib.util
import pytest
import torch
from tests.kernels.utils import opcheck
# this import statement is needed to ensure the ops are registered
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import current_platform
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = pytest.mark.skipif(
not (current_platform.is_rocm() and aiter_available),
reason="AITER ops are only available on ROCm with aiter package installed",
)
FP8_DTYPE = current_platform.fp8_dtype()
def _x(M=128, N=4096):
return torch.randn((M, N), dtype=torch.float16, device="cuda")
def test_per_tensor_quant_static_schema():
"""Static per-tensor: caller provides scale (the aliasing regression)."""
x = _x()
out = torch.empty_like(x, dtype=FP8_DTYPE)
scale = torch.ones(1, dtype=torch.float32, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_per_tensor_quant,
(out, x, scale, False),
)
def test_per_tensor_quant_dynamic_schema():
"""Dynamic per-tensor: op computes scale into the caller's buffer."""
x = _x()
out = torch.empty_like(x, dtype=FP8_DTYPE)
scale = torch.empty(1, dtype=torch.float32, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_per_tensor_quant,
(out, x, scale, True),
)
def test_per_token_quant_dynamic_schema():
"""Dynamic per-token: op computes scale into a freshly allocated buffer."""
x = _x()
opcheck(
torch.ops.vllm.rocm_aiter_per_token_quant,
(x, FP8_DTYPE, None),
)
def test_group_fp8_quant_schema():
"""Dynamic per-token-group quant."""
x = _x()
opcheck(
torch.ops.vllm.rocm_aiter_group_fp8_quant,
(x, 128),
)
@pytest.mark.parametrize("dynamic", [True, False])
def test_per_tensor_quant_matches_native(dynamic):
"""Wrapper output matches the native scaled_fp8_quant reference."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
x = _x()
if dynamic:
scale_in = None
else:
scale_in = torch.tensor([0.5], dtype=torch.float32, device="cuda")
out, scale = rocm_aiter_ops.per_tensor_quant(x, FP8_DTYPE, scale_in)
ref_out, ref_scale = ops.scaled_fp8_quant(x, scale_in)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape == ref_scale.shape
deq = out.to(torch.float32) * scale
if dynamic:
# Dynamic mode: AITER and native each compute their own scale, so their
# outputs differ and can't be compared. Just check that AITER's output
# dequantizes back to the input, within fp8 rounding error.
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
else:
# Static mode: both use the caller's scale, so the outputs must match.
assert torch.equal(scale, scale_in)
ref_deq = ref_out.to(torch.float32) * ref_scale
torch.testing.assert_close(deq, ref_deq, rtol=2e-2, atol=2e-2)
def test_per_token_quant_matches_native():
"""Per-token quant output dequantizes back to the input within FP8 error."""
torch.manual_seed(0)
x = _x()
out, scale = rocm_aiter_ops.per_token_quant(x, FP8_DTYPE)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape[0] == x.shape[0]
# Dequantize and compare to original
deq = out.to(torch.float32) * scale.view(-1, 1)
torch.testing.assert_close(deq, x.to(torch.float32), rtol=0.07, atol=5e-2)
def test_rms_norm_determinism():
"""AITER RMSNorm produces bitwise-identical results across repeated calls."""
# Import to ensure ops are registered
import vllm.kernels.aiter_ops # noqa: F401
torch.manual_seed(0)
M, N = 32, 512
x = torch.randn(M, N, dtype=torch.bfloat16, device="cuda")
weight = torch.ones(N, dtype=torch.bfloat16, device="cuda")
eps = 1e-5
reference = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
for i in range(3):
result = torch.ops.vllm_aiter.rms_norm(x, weight, eps)
assert torch.equal(reference, result), f"Run {i + 1} differs from reference"
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# This is a test for the AITER group_fp8_quant op.
# It tests if the AITER op is
# 1. correctly defined the relationship between
# implementation and fake function
# 2. can be used with torch.compile
# 3. can be used with CUDA graphs
# This file will be skipped if AITER is not installed
# and the platform is not ROCm.
import importlib.util
import pytest
import torch
# this import statement is needed to ensure the ops are registered
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms import current_platform
# Check if aiter package is installed
aiter_available = importlib.util.find_spec("aiter") is not None
pytestmark = pytest.mark.skipif(
not (current_platform.is_rocm() and aiter_available),
reason="AITER ops are only available on ROCm with aiter package installed",
)
def test_rocm_aiter_group_fp8_quant_fake_implementation():
"""Test that the fake implementation is correctly
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant."""
# Create test tensors
M = 128
N = 4096
group_size = 128
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
# Verify the op's fake implementation using torch.library.opcheck
# This checks that the fake function returns tensors with correct shapes and dtypes
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_group_fp8_quant,
(input_tensor, group_size),
test_utils=("test_faketensor",),
)
def test_rocm_aiter_group_fp8_quant_torch_compile_with_cudagraph():
"""Test that rocm_aiter_ops.group_fp8_quant
with group size 128 can be used with
torch.compile in cudagraph mode."""
# Create test tensors
M = 128
N = 4096
group_size = 128
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
# Define a function that uses the op
def group_fp8_quant_fn(x):
return rocm_aiter_ops.group_fp8_quant(x, group_size)
# Compile with cudagraph mode
compiled_fn = torch.compile(
group_fp8_quant_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
# Run eager mode
x_fp8_eager, scales_eager = group_fp8_quant_fn(input_tensor)
# Run compiled version (first run will trigger compilation)
x_fp8_compiled, scales_compiled = compiled_fn(input_tensor)
# Verify shapes match
assert x_fp8_compiled.shape == x_fp8_eager.shape
assert scales_compiled.shape == scales_eager.shape
# Verify expected shapes
assert x_fp8_compiled.shape == (M, N)
expected_scale_cols = (N + group_size - 1) // group_size
assert scales_compiled.shape == (M, expected_scale_cols)
# Verify results match
assert torch.allclose(
x_fp8_compiled.to(torch.float32),
x_fp8_eager.to(torch.float32),
rtol=1e-2,
atol=1e-2,
)
assert torch.allclose(scales_compiled, scales_eager, rtol=1e-3, atol=1e-3)
# Test with different input (reusing compiled graph)
input_tensor_2 = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
x_fp8_eager_2, scales_eager_2 = group_fp8_quant_fn(input_tensor_2)
x_fp8_compiled_2, scales_compiled_2 = compiled_fn(input_tensor_2)
# Verify second run also produces correct results
assert torch.allclose(
x_fp8_compiled_2.to(torch.float32),
x_fp8_eager_2.to(torch.float32),
rtol=1e-2,
atol=1e-2,
)
assert torch.allclose(scales_compiled_2, scales_eager_2, rtol=1e-3, atol=1e-3)
def test_rocm_aiter_group_fp8_quant_different_shapes():
"""Test rocm_aiter_ops.group_fp8_quant with different input shapes."""
group_size = 128
test_shapes = [
(64, 2048),
(256, 8192),
(32, 1024),
(512, 4096),
]
for M, N in test_shapes:
input_tensor = torch.randn((M, N), dtype=torch.bfloat16, device="cuda")
x_fp8, scales = rocm_aiter_ops.group_fp8_quant(input_tensor, group_size)
# Verify shapes
assert x_fp8.shape == (M, N)
expected_scale_cols = (N + group_size - 1) // group_size
assert scales.shape == (M, expected_scale_cols)
# Verify dtypes
from aiter import dtypes
assert x_fp8.dtype == dtypes.fp8
assert scales.dtype == torch.float32