[ROCm][CI] Add More AITER quantization/MoE kernel tests (#49375)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Micah Williamson
2026-08-05 21:27:33 +00:00
committed by GitHub
co-authored by Andreas Karatzas
parent 482c613423
commit 4282fe52d5
5 changed files with 2335 additions and 2 deletions
+20 -2
View File
@@ -1613,8 +1613,12 @@ steps:
source_file_dependencies:
- csrc/quantization/cutlass_w8a8/moe/
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe
- tests/kernels/utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/mxfp4.py
- vllm/model_executor/layers/quantization/utils/quant_utils.py
- vllm/distributed/device_communicators/
- vllm/envs.py
- vllm/config
@@ -1634,12 +1638,17 @@ steps:
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- csrc/quantization/
- csrc/rocm/
- vllm/model_executor/layers/quantization
- tests/kernels/quantization
- tests/kernels/quantization/test_rocm_skinny_gemms.py
- tests/kernels/quant_utils.py
- tests/kernels/utils.py
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/model_executor/kernels/
- vllm/v1/attention/backends/rocm_aiter_fa.py
commands:
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
@@ -3406,8 +3415,12 @@ steps:
source_file_dependencies:
- csrc/quantization/cutlass_w8a8/moe/
- csrc/moe/
- csrc/rocm/
- tests/kernels/moe
- tests/kernels/utils.py
- vllm/model_executor/layers/fused_moe/
- vllm/model_executor/layers/quantization/mxfp4.py
- vllm/model_executor/layers/quantization/utils/quant_utils.py
- vllm/distributed/device_communicators/
- vllm/envs.py
- vllm/config
@@ -3426,12 +3439,17 @@ steps:
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- csrc/quantization/
- csrc/rocm/
- vllm/model_executor/layers/quantization
- tests/kernels/quantization
- tests/kernels/quantization/test_rocm_skinny_gemms.py
- tests/kernels/quant_utils.py
- tests/kernels/utils.py
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- vllm/model_executor/kernels/
- vllm/v1/attention/backends/rocm_aiter_fa.py
commands:
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
File diff suppressed because it is too large Load Diff
+541
View File
@@ -0,0 +1,541 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm-specific tests for FP8 quantization and kernels."""
import importlib
import math
import warnings
import pytest
import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import (
ref_dynamic_per_tensor_fp8_quant,
ref_dynamic_per_token_quant,
)
from tests.kernels.utils import _assert_deterministic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
DEVICE = "cuda"
FP8_DTYPE = current_platform.fp8_dtype()
fp8_only = pytest.mark.skipif(
not current_platform.supports_fp8(), reason="ROCm FP8 only"
)
ENV_DEFAULTS = {
"VLLM_ROCM_FP8_PADDING": True,
"VLLM_ROCM_USE_AITER_FP8BMM": True,
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
}
SCALED_MM_CASES = [
pytest.param((1, 128, 128), 0.05, 0.15, 0.95, id="1x128x128"),
pytest.param((4, 256, 512), 0.10, 0.30, 0.96, id="4x256x512"),
pytest.param((8, 512, 1024), 0.20, 0.50, 0.99, id="8x512x1024"),
]
def _reload_envs():
import vllm.envs as envs
return importlib.reload(envs)
@pytest.fixture(autouse=True)
def _restore_rocm_env_state():
"""Restore global env + AITER flag state after every test.
Tests here reload ``vllm.envs`` (a module singleton) and refresh the
cached ``rocm_aiter_ops`` env flags. Restoring in a fixture teardown
guarantees cleanup even when an assertion fails mid-test, preventing
state from leaking into other tests in the same worker.
"""
yield
_reload_envs()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
def _quantile(flat: torch.Tensor, q: float) -> float:
if flat.numel() == 1:
return flat.item()
return torch.quantile(flat, q).item()
def _format_observed_rate(count: int, total: int) -> str:
return f"{count / total:.4%} ({count}/{total})"
def _format_allowed_rate(rate: float, total: int) -> str:
allowed_count = math.floor(rate * total)
return f"{rate:.4%} (<= {allowed_count}/{total})"
def _assert_abs_error_budget(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
tight_atol: float,
max_atol: float,
pass_rate: float,
) -> None:
diff = (actual.float() - expected.float()).abs().flatten()
total = diff.numel()
within_tight_count = int((diff <= tight_atol).sum().item())
fail_count = total - within_tight_count
above_max_count = int((diff > max_atol).sum().item())
within_tight = within_tight_count / total
allowed_fail_rate = 1.0 - pass_rate
above_max = above_max_count / total
max_fail_rate = 0.0
max_diff = diff.max().item()
mean_diff = diff.mean().item()
p99 = _quantile(diff, 0.99)
p999 = _quantile(diff, 0.999)
msg = (
f"[rocm_fp8] {label}: abs<={tight_atol} pass={within_tight:.4%} "
f"({within_tight_count}/{total}) "
f"fail={_format_observed_rate(fail_count, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"abs>{max_atol}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
f"max_diff={max_diff:.6g} mean_diff={mean_diff:.6g} "
f"p99={p99:.6g} p999={p999:.6g}"
)
print(msg)
if within_tight < 1.0:
warnings.warn(msg, stacklevel=2)
assert within_tight >= pass_rate, msg
assert above_max <= max_fail_rate, msg
def _assert_rel_error_budget(
actual: torch.Tensor,
expected: torch.Tensor,
*,
label: str,
tight_rtol: float,
max_rtol: float,
pass_rate: float,
max_fail_rate: float,
) -> None:
rel = (
(actual.float() - expected.float()).abs()
/ expected.float().abs().clamp_min(1e-5)
).flatten()
total = rel.numel()
within_tight_count = int((rel <= tight_rtol).sum().item())
fail_count = total - within_tight_count
above_max_count = int((rel > max_rtol).sum().item())
within_tight = within_tight_count / total
allowed_fail_rate = 1.0 - pass_rate
above_max = above_max_count / total
max_rel = rel.max().item()
mean_rel = rel.mean().item()
p99 = _quantile(rel, 0.99)
p999 = _quantile(rel, 0.999)
msg = (
f"[rocm_fp8] {label}: rel<={tight_rtol} pass={within_tight:.4%} "
f"({within_tight_count}/{total}) "
f"fail={_format_observed_rate(fail_count, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"rel>{max_rtol}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
f"max_rel={max_rel:.6g} mean_rel={mean_rel:.6g} "
f"p99={p99:.6g} p999={p999:.6g}"
)
print(msg)
if within_tight < 1.0:
warnings.warn(msg, stacklevel=2)
assert within_tight >= pass_rate, msg
assert above_max <= max_fail_rate, msg
def _ref_paged_attention(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
*,
query_lens: list[int],
kv_lens: list[int],
block_tables: torch.Tensor,
scale: float,
) -> torch.Tensor:
block_tables_np = block_tables.cpu().numpy()
_, block_size, num_kv_heads, head_size = key_cache.shape
outputs = []
start_idx = 0
for i, query_len in enumerate(query_lens):
kv_len = kv_lens[i]
q = query[start_idx : start_idx + query_len] * scale
num_kv_blocks = (kv_len + block_size - 1) // block_size
block_indices = block_tables_np[i, :num_kv_blocks]
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
if q.shape[1] != k.shape[1]:
repeat = q.shape[1] // k.shape[1]
k = torch.repeat_interleave(k, repeat, dim=1)
v = torch.repeat_interleave(v, repeat, dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
mask = torch.triu(
torch.ones(query_len, kv_len, device=q.device),
diagonal=kv_len - query_len + 1,
).bool()
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1).to(v.dtype)
outputs.append(torch.einsum("hqk,khd->qhd", attn, v))
start_idx += query_len
return torch.cat(outputs, dim=0)
def test_rocm_fp8_env_defaults(monkeypatch):
with monkeypatch.context() as mp:
for env_name in ENV_DEFAULTS:
mp.delenv(env_name, raising=False)
envs = _reload_envs()
for env_name, expected in ENV_DEFAULTS.items():
assert getattr(envs, env_name) is expected
@pytest.mark.parametrize(
("env_name", "raw_value", "expected"),
[
("VLLM_ROCM_FP8_PADDING", "0", False),
("VLLM_ROCM_FP8_PADDING", "1", True),
("VLLM_ROCM_USE_AITER_FP8BMM", "0", False),
("VLLM_ROCM_USE_AITER_FP8BMM", "1", True),
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0", False),
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "1", True),
],
)
def test_rocm_fp8_env_overrides(env_name, raw_value, expected, monkeypatch):
with monkeypatch.context() as mp:
mp.setenv(env_name, raw_value)
envs = _reload_envs()
assert getattr(envs, env_name) is expected
@fp8_only
def test_maybe_pad_fp8_weight_respects_env(monkeypatch):
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_maybe_pad_fp8_weight,
)
weight = torch.randn(64, 512, device=DEVICE).to(FP8_DTYPE)
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_FP8_PADDING", "1")
_reload_envs()
padded = _maybe_pad_fp8_weight(weight)
assert padded.shape == weight.shape
assert padded.dtype == weight.dtype
assert padded.data_ptr() != weight.data_ptr()
torch.testing.assert_close(padded.float(), weight.float())
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_FP8_PADDING", "0")
_reload_envs()
unpadded = _maybe_pad_fp8_weight(weight)
assert unpadded.data_ptr() == weight.data_ptr()
@fp8_only
@pytest.mark.parametrize(
("use_aiter", "fp8bmm_enabled", "expected"),
[
(True, True, True),
(True, False, False),
(False, True, False),
],
)
def test_aiter_fp8bmm_enabled_api_respects_env(
use_aiter, fp8bmm_enabled, expected, monkeypatch
):
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
assert is_aiter_found_and_supported()
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv("VLLM_ROCM_USE_AITER_FP8BMM", "1" if fp8bmm_enabled else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert rocm_aiter_ops.is_fp8bmm_enabled() is expected
@fp8_only
@pytest.mark.parametrize(
("shape", "tight_atol", "max_atol", "pass_rate"), SCALED_MM_CASES
)
def test_fp8_scaled_mm_matches_dequantized_reference(
shape, tight_atol, max_atol, pass_rate
):
torch.manual_seed(0)
batch, m_dim, k_dim = shape
n_dim = k_dim
a = torch.randn(batch * m_dim, k_dim, device=DEVICE)
b = torch.randn(n_dim, k_dim, device=DEVICE)
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
out = torch._scaled_mm(
a_fp8,
b_fp8.t(),
out_dtype=torch.bfloat16,
scale_a=scale_a,
scale_b=scale_b,
)
a_dq = a_fp8.float() * scale_a.float()
b_dq = b_fp8.float() * scale_b.float()
ref = torch.mm(a_dq, b_dq.t())
assert out.shape == (batch * m_dim, n_dim)
assert out.dtype == torch.bfloat16
assert not torch.isnan(out).any()
_assert_abs_error_budget(
out.float(),
ref,
label=f"scaled_mm shape={shape}",
tight_atol=tight_atol,
max_atol=max_atol,
pass_rate=pass_rate,
)
@fp8_only
def test_fp8_scaled_mm_determinism():
torch.manual_seed(5)
m_dim, k_dim, n_dim = 64, 128, 128
a = torch.randn(m_dim, k_dim, device=DEVICE)
b = torch.randn(n_dim, k_dim, device=DEVICE)
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
def run_gemm():
return torch._scaled_mm(
a_fp8,
b_fp8.t(),
out_dtype=torch.bfloat16,
scale_a=scale_a,
scale_b=scale_b,
)
_assert_deterministic(run_gemm, n_runs=4)
@fp8_only
def test_aiter_fp8_kv_attention_matches_bf16_reference():
import aiter
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.v1.attention.backends.rocm_aiter_fa import cp_mha_gather_cache
assert is_aiter_found_and_supported()
set_random_seed(10)
num_q_heads, num_kv_heads = 8, 8
head_size = 128
query_len, kv_len = 4, 128
block_size = 16
num_blocks = 2048
scale = head_size**-0.5
query = torch.randn(
query_len, num_q_heads, head_size, dtype=torch.bfloat16, device=DEVICE
)
key_cache_fp8 = torch.clamp(
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
min=-1.0,
max=1.0,
).to(FP8_DTYPE)
value_cache_fp8 = torch.clamp(
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
min=-1.0,
max=1.0,
).to(FP8_DTYPE)
cu_query_lens = torch.tensor([0, query_len], dtype=torch.int32, device=DEVICE)
cu_seq_lens = torch.tensor([0, kv_len], dtype=torch.int32, device=DEVICE)
block_tables = torch.randint(
0,
num_blocks,
(1, (kv_len + block_size - 1) // block_size),
dtype=torch.int32,
device=DEVICE,
)
token_to_batch = torch.zeros(kv_len, dtype=torch.int32, device=DEVICE)
seq_starts = torch.zeros(1, dtype=torch.int32, device=DEVICE)
gathered_key = torch.empty(
kv_len, num_kv_heads, head_size, dtype=torch.bfloat16, device=DEVICE
)
gathered_value = torch.empty_like(gathered_key)
cp_mha_gather_cache(
key_cache=key_cache_fp8,
value_cache=value_cache_fp8,
key=gathered_key,
value=gathered_value,
block_tables=block_tables,
k_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
v_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
cu_seqlens_kv=cu_seq_lens,
token_to_batch=token_to_batch,
seq_starts=seq_starts,
dequant=True,
kv_cache_layout="NHD",
total_tokens=kv_len,
)
output = torch.empty_like(query)
aiter.flash_attn_varlen_func(
q=query,
k=gathered_key,
v=gathered_value,
cu_seqlens_q=cu_query_lens,
cu_seqlens_k=cu_seq_lens,
max_seqlen_q=query_len,
max_seqlen_k=kv_len,
min_seqlen_q=1,
dropout_p=0.0,
softmax_scale=scale,
causal=True,
window_size=(-1, -1),
alibi_slopes=None,
return_lse=False,
out=output,
)
ref = _ref_paged_attention(
query=query,
key_cache=key_cache_fp8.to(torch.bfloat16),
value_cache=value_cache_fp8.to(torch.bfloat16),
query_lens=[query_len],
kv_lens=[kv_len],
block_tables=block_tables,
scale=scale,
)
_assert_abs_error_budget(
output.float(),
ref.float(),
label="aiter_fp8_kv_attention",
tight_atol=0.15,
max_atol=0.25,
pass_rate=0.98,
)
@fp8_only
@pytest.mark.parametrize("shape", [(128, 256), (512, 1024), (1024, 4096)])
def test_fp8_per_tensor_quant_matches_reference(shape):
torch.manual_seed(0)
x = torch.randn(*shape, dtype=torch.bfloat16, device=DEVICE)
out, scale = ops.scaled_fp8_quant(x)
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
code_match_rate = (
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
)
assert code_match_rate >= 0.999, (
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
)
dequant = out.float() * scale.float()
_assert_rel_error_budget(
dequant,
x.float(),
label=f"per_tensor_quant shape={shape}",
tight_rtol=0.06,
max_rtol=0.60,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@fp8_only
def test_aiter_group_fp8_quant_roundtrip():
from aiter import dtypes
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
assert is_aiter_found_and_supported()
torch.manual_seed(0)
m_dim, n_dim = 64, 4096
group_size = 128
x = torch.randn(m_dim, n_dim, dtype=torch.bfloat16, device=DEVICE)
out, scales = rocm_aiter_ops.group_fp8_quant(x, group_size)
assert out.dtype == dtypes.fp8
assert scales.dtype == torch.float32
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :n_dim]
dequant = out.float() * scales_expanded
_assert_rel_error_budget(
dequant,
x.float(),
label="aiter_group_fp8_quant",
tight_rtol=0.06,
max_rtol=0.30,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@fp8_only
def test_fp8_per_token_quant_matches_reference():
torch.manual_seed(0)
x = torch.randn(32, 4096, dtype=torch.bfloat16, device=DEVICE)
out, scale = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
ref_out, ref_scale = ref_dynamic_per_token_quant(x, FP8_DTYPE)
assert out.shape == x.shape
assert out.dtype == FP8_DTYPE
assert scale.shape == (x.shape[0], 1)
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
code_match_rate = (
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
)
assert code_match_rate >= 0.999, (
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
)
dequant = out.float() * scale.float()
_assert_rel_error_budget(
dequant,
x.float(),
label="per_token_quant",
tight_rtol=0.06,
max_rtol=0.30,
pass_rate=0.99,
max_fail_rate=1e-4,
)
@@ -0,0 +1,638 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm AITER FP4 and MXFP4 tests.
This file keeps the ROCm-specific FP4 coverage:
- ROCm AITER FP4 env and enablement gates
- large-shape MXFP4 wrapper checks through vLLM's public helper op
- AITER Triton MXFP4 quant format and determinism
- FP4 GEMM, preshuffled-scale, hardware-quant, and skinny decode
paths
Generic MXFP4 wrapper/reference checks already live in
``tests/kernels/quantization/test_quark.py``.
"""
import importlib
import warnings
import pytest
import torch
from tests.kernels.utils import _assert_accurate, _assert_deterministic
from tests.quantization.reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
LLAMA_SHAPES = [
(64, 4096),
(64, 11008),
(32, 14336),
]
GEMM_ATOL = 0.5
GEMM_RTOL = 0.0
GEMM_MAX_VIOLATION_FACTOR = 2.0
A4W4_GEMM_PASS_RATES = {
(64, 128, 64): 0.9999,
(128, 256, 128): 0.9999,
(128, 4096, 4096): 0.9999,
(256, 4096, 11008): 0.9999,
(64, 8192, 28672): 0.99765,
}
SKINNY_GEMM_PASS_RATES = {
(4096, 4096): 0.9999,
(4096, 11008): 0.99235,
(8192, 8192): 0.99765,
}
PRESHUFFLED_SHAPES = [
(64, 4096, 8192),
(32, 8192, 8192),
]
def _reload_envs():
import vllm.envs as envs
return importlib.reload(envs)
@pytest.fixture(autouse=True)
def _restore_rocm_env_state():
"""Restore global env + AITER flag state after every test."""
yield
_reload_envs()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.refresh_env_variables()
def _assert_aiter_supported() -> None:
from vllm._aiter_ops import is_aiter_found_and_supported
assert is_aiter_found_and_supported(), (
"aiter is required on supported ROCm hardware for this test"
)
def _format_observed_rate(count: int, total: int) -> str:
return f"{count / total:.4%} ({count}/{total})"
def _format_allowed_rate(rate: float, total: int) -> str:
allowed_count = int(rate * total)
return f"{rate:.4%} (<= {allowed_count}/{total})"
def _print_close_stats(
label: str,
actual: torch.Tensor,
expected: torch.Tensor,
*,
atol: float,
rtol: float,
pass_rate: float = 1.0,
max_atol: float | None = None,
) -> None:
abs_diff = (actual - expected).abs().float().flatten()
expected_abs = expected.abs().float().flatten()
allowed = atol + rtol * expected_abs
within = abs_diff <= allowed
total = abs_diff.numel()
passed = int(within.sum().item())
failed = total - passed
allowed_fail_rate = 1.0 - pass_rate
max_abs = abs_diff.max().item()
mean_abs = abs_diff.mean().item()
p99_abs = torch.quantile(abs_diff, 0.99).item()
p999_abs = torch.quantile(abs_diff, 0.999).item()
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
msg = (
"[rocm_aiter_fp4] "
f"{label}: "
f"pass={passed / total:.4%} ({passed}/{total}) "
f"fail={_format_observed_rate(failed, total)} "
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
f"atol={atol:g} "
f"rtol={rtol:g} "
)
if max_atol is not None:
above_max_count = int((abs_diff > max_atol).sum().item())
msg += (
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
)
msg += (
f"max_abs={max_abs:.6g} "
f"mean_abs={mean_abs:.6g} "
f"p99_abs={p99_abs:.6g} "
f"p999_abs={p999_abs:.6g} "
f"worst_ratio={worst_ratio:.6g}"
)
print(msg)
if failed > 0:
warnings.warn(msg, stacklevel=2)
# Env and enablement tests ------------------------------------------------
def test_fp4_env_defaults():
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True
@pytest.mark.parametrize(
(
"use_aiter",
"use_fp4_asm_gemm",
"use_fp4bmm",
),
[
(True, True, True),
(True, True, False),
(True, False, True),
(False, True, True),
],
)
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
use_aiter,
use_fp4_asm_gemm,
use_fp4bmm,
monkeypatch,
):
"""The ROCm FP4 AITER gates should depend only on the env toggles and the
gfx950 hardware check.
Expected values are derived from the *actual* platform rather than a patched
arch (gfx950 -> MI355, gfx942 -> MI300). A failure in this group therefore
points straight at the hardware it ran on instead of a mocked value.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
on_gfx950_value = on_gfx950()
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv(
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
"1" if use_fp4_asm_gemm else "0",
)
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert (
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
)
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
# Large-shape MXFP4 wrapper tests -----------------------------------------
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
reference even on larger Llama-like shapes."""
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
rows, cols = shape
x = torch.randn(rows, cols, dtype=dtype)
out = quant_dequant_mxfp4(x)
ref = qdq_mxfp4_torch(x, "even")
assert out.shape == x.shape
assert out.dtype == dtype
assert torch.isfinite(out).all()
torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0)
def test_vllm_quant_dequant_mxfp4_is_deterministic():
"""The public vLLM MXFP4 QDQ helper should stay bitwise deterministic."""
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(3)
x = torch.randn(64, 128, dtype=torch.bfloat16)
_assert_deterministic(quant_dequant_mxfp4, x, n_runs=4)
# Triton MXFP4 quant tests ------------------------------------------------
def test_aiter_dynamic_mxfp4_quant_output_format():
"""dynamic_mxfp4_quant returns packed uint8 FP4 values and E8M0 uint8 scales.
OCP MXFP4: block_size=32, 2 FP4 E2M1 values packed per byte.
Scale shape: (M, K // 32); one E8M0 exponent byte per 32-element block.
"""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
M, K = 64, 256
x = torch.randn(M, K, dtype=torch.bfloat16)
x_fp4, x_scale = dynamic_mxfp4_quant(x)
# FP4 values packed 2-per-byte; shape (M, K // 2)
assert x_fp4.shape == (M, K // 2), (
f"Expected fp4 shape ({M}, {K // 2}), got {x_fp4.shape}"
)
assert x_fp4.dtype == torch.uint8
# One E8M0 scale byte per 32-element block; shape (M, K // 32)
assert x_scale.shape == (M, K // 32), (
f"Expected scale shape ({M}, {K // 32}), got {x_scale.shape}"
)
assert x_scale.dtype == torch.uint8
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
"shape",
[
(128, 4096), # Llama-7B hidden
(256, 11008), # Llama-7B FFN
(64, 14336), # Llama-70B FFN
(512, 8192), # Llama-70B hidden
(32, 28672), # DeepSeek-style large FFN
],
)
def test_aiter_dynamic_mxfp4_quant_llama_shapes(shape, dtype):
"""MXFP4 quantization output format is correct for Llama-class weight shapes.
Targets B200 parity: NVIDIA nvfp4 tests parametrize over Llama shapes
(7168, 14336, 28672...). This covers gfx950 MXFP4 quant format validation.
"""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
M, K = shape
x = torch.randn(M, K, dtype=dtype)
x_fp4, x_scale = dynamic_mxfp4_quant(x)
assert x_fp4.shape == (M, K // 2)
assert x_fp4.dtype == torch.uint8
assert x_scale.shape == (M, K // 32)
assert x_scale.dtype == torch.uint8
assert not torch.any(torch.isnan(x_fp4.float()))
def test_aiter_dynamic_mxfp4_quant_determinism():
"""dynamic_mxfp4_quant is bitwise deterministic across repeated runs."""
_assert_aiter_supported()
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(7)
x = torch.randn(128, 256, dtype=torch.bfloat16)
_assert_deterministic(dynamic_mxfp4_quant, x, n_runs=4)
# gfx950 hardware FP4 GEMM tests ------------------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
[
(64, 128, 64), # small square-ish
(128, 256, 128), # medium
(128, 4096, 4096), # Llama-7B hidden square
(256, 4096, 11008), # Llama-7B FFN
(64, 8192, 28672), # Llama-70B FFN
],
)
def test_aiter_fp4_gemm_a4w4_accuracy(shape):
"""AITER A4W4 FP4 GEMM output is close to matmul on dequantized weights.
Tests gfx950 parity with B200 test_nvfp4_scaled_mm.py: E2M1 x E2M1 GEMM
with block-scaled quantization, multiple Llama-class shapes.
Requires gfx950 hardware; gracefully skips elsewhere.
"""
atol = GEMM_ATOL
rtol = GEMM_RTOL
pass_rate = A4W4_GEMM_PASS_RATES[shape]
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
M, K, N = shape
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16) # weight stored as (N, K)
# Quantize
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
# FP4 GEMM computes A_fp4 (M, K) @ B_fp4.T (K, N) into (M, N).
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
assert out.shape == (M, N), f"Expected ({M}, {N}), got {out.shape}"
assert out.dtype == torch.bfloat16
assert not torch.any(torch.isnan(out))
# Reference: matmul on dequantized FP4 inputs (quark roundtrip)
A_dq = quant_dequant_mxfp4(A)
B_dq = quant_dequant_mxfp4(B)
ref = torch.matmul(A_dq.float(), B_dq.t().float())
# FP4 has only 3 mantissa bits; large K amplifies the accumulation error.
# These budgets were measured on gfx950 and keep p99/p999 drift visible
# without pretending the kernel is close to BF16 matmul everywhere.
_print_close_stats(
f"a4w4_gemm shape={shape}",
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
)
_assert_accurate(
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
PRESHUFFLED_SHAPES,
)
def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
"""The preshuffled FP4 GEMM path should run on tuned shapes with the same
packed inputs vLLM uses in production.
This kernel only advertises a narrow tuned `(N, K)` set on gfx950. The
important contract here is that those tuned shapes execute with the
production-style shuffled weights and stay deterministic.
"""
_assert_aiter_supported()
from aiter import per_1x32_f4_quant_hip
from aiter.ops.shuffle import shuffle_weight
from aiter.ops.triton.gemm_afp4wfp4 import (
gemm_afp4wfp4_preshuffled_weight_scales,
)
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm._aiter_ops import rocm_aiter_ops
torch.set_default_device("cuda")
torch.manual_seed(1)
M, K, N = shape
assert M <= 64
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
A_q, A_s = per_1x32_f4_quant_hip(A, shuffle=M >= 32)
scale_rows, scale_cols = B_scale.shape
B_scale = (
B_scale.view(scale_rows // 32, 2, 16, scale_cols // 8, 2, 4, 1)
.permute(0, 3, 5, 2, 4, 1, 6)
.contiguous()
.view(scale_rows, scale_cols)
)
B_fp4 = shuffle_weight(B_fp4, layout=(16, 16))
if M >= 32:
A_s = A_s.contiguous().view(torch.uint8).reshape(A_s.shape[0] // 32, -1)
else:
A_s = A_s[:M, ...].contiguous().view(torch.uint8)
B_scale = B_scale.contiguous().view(torch.uint8).reshape(B_scale.shape[0] // 32, -1)
def run_preshuffled() -> torch.Tensor:
y = torch.empty(M, N, device="cuda", dtype=torch.bfloat16)
return gemm_afp4wfp4_preshuffled_weight_scales(
A_q.contiguous().view(torch.uint8),
B_fp4.contiguous().view(torch.uint8).reshape(B_fp4.shape[0] // 16, -1),
A_s,
B_scale,
torch.bfloat16,
y,
)
out = run_preshuffled()
assert out.shape == (M, N)
assert out.dtype == torch.bfloat16
assert torch.isfinite(out).all()
_assert_deterministic(run_preshuffled, n_runs=3)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_gemm_a4w4_determinism():
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
Targets parity with B200 FP4 determinism requirements.
Requires gfx950.
"""
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(2)
M, K, N = 128, 256, 128
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
def run_gemm():
return gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
_assert_deterministic(run_gemm, n_runs=4)
# gfx950 hardware FP4 dynamic quantization --------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize(
"shape",
[
(128, 256),
(256, 4096),
(64, 14336),
],
)
def test_aiter_hardware_fp4_dynamic_quant_format(shape):
"""aiter hardware FP4 dynamic quant produces correct output format.
Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1).
Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output.
Requires gfx950.
"""
_assert_aiter_supported()
from aiter import dynamic_per_group_scaled_quant_fp4
torch.set_default_device("cuda")
M, K = shape
group_size = 32
x = torch.randn(M, K, dtype=torch.bfloat16)
out_fp4 = torch.empty(M, K // 2, dtype=torch.uint8)
scales = torch.empty(M, K // group_size, dtype=torch.uint8)
dynamic_per_group_scaled_quant_fp4(out_fp4, x, scales, group_size)
assert out_fp4.shape == (M, K // 2), (
f"Shape {shape}: expected fp4 ({M}, {K // 2}), got {out_fp4.shape}"
)
assert scales.shape == (M, K // group_size), (
f"Shape {shape}: expected scale ({M}, {K // group_size}), got {scales.shape}"
)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_hardware_fp4_quant_vs_triton():
"""Hardware FP4 quant should dequantize back to the same MXFP4 values as
the Triton path.
The packed bytes are not identical on gfx950, but with
``shuffle_scale=False`` the dequantized results match tightly.
"""
_assert_aiter_supported()
from aiter import dynamic_per_group_scaled_quant_fp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
torch.set_default_device("cuda")
torch.manual_seed(5)
M, K = 128, 256
group_size = 32
x = torch.randn(M, K, dtype=torch.bfloat16)
# Hardware path
out_hw = torch.empty(M, K // 2, dtype=torch.uint8)
scales_hw = torch.empty(M, K // group_size, dtype=torch.uint8)
dynamic_per_group_scaled_quant_fp4(
out_hw, x, scales_hw, group_size, shuffle_scale=False
)
# Triton path
out_triton, scales_triton = dynamic_mxfp4_quant(x)
assert out_hw.shape == out_triton.shape
assert scales_hw.shape == scales_triton.shape
dq_hw = dq_mxfp4_torch(out_hw, scales_hw, torch.bfloat16)
dq_triton = dq_mxfp4_torch(out_triton, scales_triton, torch.bfloat16)
_print_close_stats(
"hardware_quant_vs_triton",
dq_hw.float(),
dq_triton.float(),
atol=0.25,
rtol=0.0,
)
torch.testing.assert_close(dq_hw, dq_triton, atol=0.25, rtol=0.0)
# ROCm skinny GEMM FP4 tests ----------------------------------------------
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
@pytest.mark.parametrize("M", [1, 2, 4, 8]) # decode / skinny batch sizes
@pytest.mark.parametrize("N, K", [(4096, 4096), (4096, 11008), (8192, 8192)])
def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
"""FP4 GEMM accuracy for skinny (small-M) shapes (decode phase).
Skinny GEMMs (M=1..8) are the bottleneck in decode phase.
Tests parity with B200 nvfp4_scaled_mm tests at decode batch sizes.
Requires gfx950.
"""
atol = GEMM_ATOL
rtol = GEMM_RTOL
pass_rate = SKINNY_GEMM_PASS_RATES[(N, K)]
_assert_aiter_supported()
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
torch.set_default_device("cuda")
torch.manual_seed(0)
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
A_fp4, A_scale = dynamic_mxfp4_quant(A)
B_fp4, B_scale = dynamic_mxfp4_quant(B)
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
assert out.shape == (M, N)
assert not torch.any(torch.isnan(out))
A_dq = quant_dequant_mxfp4(A)
B_dq = quant_dequant_mxfp4(B)
ref = torch.matmul(A_dq.float(), B_dq.t().float())
_print_close_stats(
f"skinny_gemm M={M} N={N} K={K}",
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
)
_assert_accurate(
out.float(),
ref,
atol=atol,
rtol=rtol,
pass_rate=pass_rate,
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
)
+57
View File
@@ -36,6 +36,63 @@ ALL_OPCHECK_TEST_UTILS: tuple[str, ...] = (
)
def _assert_accurate(
actual: torch.Tensor,
expected: torch.Tensor,
atol: float,
rtol: float = 0.0,
pass_rate: float = 0.99999,
max_violation_factor: float = 3.0,
) -> None:
"""Check numeric accuracy with pass-rate, max-error, and mean-error bounds."""
a = actual.detach().float().flatten()
e = expected.detach().float().flatten()
abs_err = (a - e).abs()
tol = atol + rtol * e.abs()
rate = (abs_err <= tol).float().mean().item()
assert rate >= pass_rate, (
f"Accuracy pass rate {rate:.6f} < {pass_rate} (atol={atol}, rtol={rtol})"
)
max_err = abs_err.max().item()
assert max_err <= max_violation_factor * atol, (
f"Max absolute error {max_err:.6f} exceeds {max_violation_factor} * atol={atol}"
)
mean_err = abs_err.mean().item()
assert mean_err <= atol * 0.25, (
f"Mean absolute error {mean_err:.6f} >= atol * 0.25 = {atol * 0.25:.6f}"
)
def _assert_deterministic(
fn,
*args,
n_runs: int = 4,
**kwargs,
) -> None:
"""Verify that repeated calls produce bitwise-identical tensor outputs."""
def _collect(result: Any) -> list[torch.Tensor]:
if isinstance(result, torch.Tensor):
return [result.detach().clone()]
if isinstance(result, (tuple, list)):
return [t.detach().clone() for t in result if isinstance(t, torch.Tensor)]
raise TypeError(f"Unexpected return type {type(result)}")
reference = _collect(fn(*args, **kwargs))
for run in range(1, n_runs):
outputs = _collect(fn(*args, **kwargs))
for idx, (ref, out) in enumerate(zip(reference, outputs)):
assert torch.equal(ref, out), (
f"Run {run}: output[{idx}] differs from run 0 "
f"(max diff = {(out.float() - ref.float()).abs().max().item():.2e})"
)
class QKVInputs(NamedTuple):
"""
Data structure for representing unpacked attention inputs,