[ROCm][CI] Add More AITER quantization/MoE kernel tests (#49375)
Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
482c613423
commit
4282fe52d5
@@ -1613,8 +1613,12 @@ steps:
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- csrc/moe/
|
||||
- csrc/rocm/
|
||||
- tests/kernels/moe
|
||||
- tests/kernels/utils.py
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/mxfp4.py
|
||||
- vllm/model_executor/layers/quantization/utils/quant_utils.py
|
||||
- vllm/distributed/device_communicators/
|
||||
- vllm/envs.py
|
||||
- vllm/config
|
||||
@@ -1634,12 +1638,17 @@ steps:
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- csrc/rocm/
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/kernels/quantization
|
||||
- tests/kernels/quantization/test_rocm_skinny_gemms.py
|
||||
- tests/kernels/quant_utils.py
|
||||
- tests/kernels/utils.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/_custom_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/model_executor/kernels/
|
||||
- vllm/v1/attention/backends/rocm_aiter_fa.py
|
||||
commands:
|
||||
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
@@ -3406,8 +3415,12 @@ steps:
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- csrc/moe/
|
||||
- csrc/rocm/
|
||||
- tests/kernels/moe
|
||||
- tests/kernels/utils.py
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
- vllm/model_executor/layers/quantization/mxfp4.py
|
||||
- vllm/model_executor/layers/quantization/utils/quant_utils.py
|
||||
- vllm/distributed/device_communicators/
|
||||
- vllm/envs.py
|
||||
- vllm/config
|
||||
@@ -3426,12 +3439,17 @@ steps:
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- csrc/rocm/
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/kernels/quantization
|
||||
- tests/kernels/quantization/test_rocm_skinny_gemms.py
|
||||
- tests/kernels/quant_utils.py
|
||||
- tests/kernels/utils.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/_custom_ops.py
|
||||
- vllm/envs.py
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/model_executor/kernels/
|
||||
- vllm/v1/attention/backends/rocm_aiter_fa.py
|
||||
commands:
|
||||
- pytest -v -s kernels/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,541 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm-specific tests for FP8 quantization and kernels."""
|
||||
|
||||
import importlib
|
||||
import math
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import (
|
||||
ref_dynamic_per_tensor_fp8_quant,
|
||||
ref_dynamic_per_token_quant,
|
||||
)
|
||||
from tests.kernels.utils import _assert_deterministic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
DEVICE = "cuda"
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
fp8_only = pytest.mark.skipif(
|
||||
not current_platform.supports_fp8(), reason="ROCm FP8 only"
|
||||
)
|
||||
|
||||
ENV_DEFAULTS = {
|
||||
"VLLM_ROCM_FP8_PADDING": True,
|
||||
"VLLM_ROCM_USE_AITER_FP8BMM": True,
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
|
||||
}
|
||||
|
||||
SCALED_MM_CASES = [
|
||||
pytest.param((1, 128, 128), 0.05, 0.15, 0.95, id="1x128x128"),
|
||||
pytest.param((4, 256, 512), 0.10, 0.30, 0.96, id="4x256x512"),
|
||||
pytest.param((8, 512, 1024), 0.20, 0.50, 0.99, id="8x512x1024"),
|
||||
]
|
||||
|
||||
|
||||
def _reload_envs():
|
||||
import vllm.envs as envs
|
||||
|
||||
return importlib.reload(envs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_rocm_env_state():
|
||||
"""Restore global env + AITER flag state after every test.
|
||||
|
||||
Tests here reload ``vllm.envs`` (a module singleton) and refresh the
|
||||
cached ``rocm_aiter_ops`` env flags. Restoring in a fixture teardown
|
||||
guarantees cleanup even when an assertion fails mid-test, preventing
|
||||
state from leaking into other tests in the same worker.
|
||||
"""
|
||||
yield
|
||||
_reload_envs()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _quantile(flat: torch.Tensor, q: float) -> float:
|
||||
if flat.numel() == 1:
|
||||
return flat.item()
|
||||
return torch.quantile(flat, q).item()
|
||||
|
||||
|
||||
def _format_observed_rate(count: int, total: int) -> str:
|
||||
return f"{count / total:.4%} ({count}/{total})"
|
||||
|
||||
|
||||
def _format_allowed_rate(rate: float, total: int) -> str:
|
||||
allowed_count = math.floor(rate * total)
|
||||
return f"{rate:.4%} (<= {allowed_count}/{total})"
|
||||
|
||||
|
||||
def _assert_abs_error_budget(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
tight_atol: float,
|
||||
max_atol: float,
|
||||
pass_rate: float,
|
||||
) -> None:
|
||||
diff = (actual.float() - expected.float()).abs().flatten()
|
||||
total = diff.numel()
|
||||
within_tight_count = int((diff <= tight_atol).sum().item())
|
||||
fail_count = total - within_tight_count
|
||||
above_max_count = int((diff > max_atol).sum().item())
|
||||
within_tight = within_tight_count / total
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
above_max = above_max_count / total
|
||||
max_fail_rate = 0.0
|
||||
max_diff = diff.max().item()
|
||||
mean_diff = diff.mean().item()
|
||||
p99 = _quantile(diff, 0.99)
|
||||
p999 = _quantile(diff, 0.999)
|
||||
msg = (
|
||||
f"[rocm_fp8] {label}: abs<={tight_atol} pass={within_tight:.4%} "
|
||||
f"({within_tight_count}/{total}) "
|
||||
f"fail={_format_observed_rate(fail_count, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"abs>{max_atol}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
|
||||
f"max_diff={max_diff:.6g} mean_diff={mean_diff:.6g} "
|
||||
f"p99={p99:.6g} p999={p999:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if within_tight < 1.0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
assert within_tight >= pass_rate, msg
|
||||
assert above_max <= max_fail_rate, msg
|
||||
|
||||
|
||||
def _assert_rel_error_budget(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
label: str,
|
||||
tight_rtol: float,
|
||||
max_rtol: float,
|
||||
pass_rate: float,
|
||||
max_fail_rate: float,
|
||||
) -> None:
|
||||
rel = (
|
||||
(actual.float() - expected.float()).abs()
|
||||
/ expected.float().abs().clamp_min(1e-5)
|
||||
).flatten()
|
||||
total = rel.numel()
|
||||
within_tight_count = int((rel <= tight_rtol).sum().item())
|
||||
fail_count = total - within_tight_count
|
||||
above_max_count = int((rel > max_rtol).sum().item())
|
||||
within_tight = within_tight_count / total
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
above_max = above_max_count / total
|
||||
max_rel = rel.max().item()
|
||||
mean_rel = rel.mean().item()
|
||||
p99 = _quantile(rel, 0.99)
|
||||
p999 = _quantile(rel, 0.999)
|
||||
msg = (
|
||||
f"[rocm_fp8] {label}: rel<={tight_rtol} pass={within_tight:.4%} "
|
||||
f"({within_tight_count}/{total}) "
|
||||
f"fail={_format_observed_rate(fail_count, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"rel>{max_rtol}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(max_fail_rate, total)} "
|
||||
f"max_rel={max_rel:.6g} mean_rel={mean_rel:.6g} "
|
||||
f"p99={p99:.6g} p999={p999:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if within_tight < 1.0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
assert within_tight >= pass_rate, msg
|
||||
assert above_max <= max_fail_rate, msg
|
||||
|
||||
|
||||
def _ref_paged_attention(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
*,
|
||||
query_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
block_tables: torch.Tensor,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
block_tables_np = block_tables.cpu().numpy()
|
||||
_, block_size, num_kv_heads, head_size = key_cache.shape
|
||||
|
||||
outputs = []
|
||||
start_idx = 0
|
||||
for i, query_len in enumerate(query_lens):
|
||||
kv_len = kv_lens[i]
|
||||
q = query[start_idx : start_idx + query_len] * scale
|
||||
|
||||
num_kv_blocks = (kv_len + block_size - 1) // block_size
|
||||
block_indices = block_tables_np[i, :num_kv_blocks]
|
||||
|
||||
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
|
||||
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
|
||||
|
||||
if q.shape[1] != k.shape[1]:
|
||||
repeat = q.shape[1] // k.shape[1]
|
||||
k = torch.repeat_interleave(k, repeat, dim=1)
|
||||
v = torch.repeat_interleave(v, repeat, dim=1)
|
||||
|
||||
attn = torch.einsum("qhd,khd->hqk", q, k).float()
|
||||
mask = torch.triu(
|
||||
torch.ones(query_len, kv_len, device=q.device),
|
||||
diagonal=kv_len - query_len + 1,
|
||||
).bool()
|
||||
attn.masked_fill_(mask, float("-inf"))
|
||||
attn = torch.softmax(attn, dim=-1).to(v.dtype)
|
||||
outputs.append(torch.einsum("hqk,khd->qhd", attn, v))
|
||||
start_idx += query_len
|
||||
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
def test_rocm_fp8_env_defaults(monkeypatch):
|
||||
with monkeypatch.context() as mp:
|
||||
for env_name in ENV_DEFAULTS:
|
||||
mp.delenv(env_name, raising=False)
|
||||
envs = _reload_envs()
|
||||
for env_name, expected in ENV_DEFAULTS.items():
|
||||
assert getattr(envs, env_name) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_name", "raw_value", "expected"),
|
||||
[
|
||||
("VLLM_ROCM_FP8_PADDING", "0", False),
|
||||
("VLLM_ROCM_FP8_PADDING", "1", True),
|
||||
("VLLM_ROCM_USE_AITER_FP8BMM", "0", False),
|
||||
("VLLM_ROCM_USE_AITER_FP8BMM", "1", True),
|
||||
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0", False),
|
||||
("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "1", True),
|
||||
],
|
||||
)
|
||||
def test_rocm_fp8_env_overrides(env_name, raw_value, expected, monkeypatch):
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv(env_name, raw_value)
|
||||
envs = _reload_envs()
|
||||
assert getattr(envs, env_name) is expected
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_maybe_pad_fp8_weight_respects_env(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_maybe_pad_fp8_weight,
|
||||
)
|
||||
|
||||
weight = torch.randn(64, 512, device=DEVICE).to(FP8_DTYPE)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_FP8_PADDING", "1")
|
||||
_reload_envs()
|
||||
padded = _maybe_pad_fp8_weight(weight)
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.dtype == weight.dtype
|
||||
assert padded.data_ptr() != weight.data_ptr()
|
||||
torch.testing.assert_close(padded.float(), weight.float())
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_FP8_PADDING", "0")
|
||||
_reload_envs()
|
||||
unpadded = _maybe_pad_fp8_weight(weight)
|
||||
assert unpadded.data_ptr() == weight.data_ptr()
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize(
|
||||
("use_aiter", "fp8bmm_enabled", "expected"),
|
||||
[
|
||||
(True, True, True),
|
||||
(True, False, False),
|
||||
(False, True, False),
|
||||
],
|
||||
)
|
||||
def test_aiter_fp8bmm_enabled_api_respects_env(
|
||||
use_aiter, fp8bmm_enabled, expected, monkeypatch
|
||||
):
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP8BMM", "1" if fp8bmm_enabled else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
assert rocm_aiter_ops.is_fp8bmm_enabled() is expected
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "tight_atol", "max_atol", "pass_rate"), SCALED_MM_CASES
|
||||
)
|
||||
def test_fp8_scaled_mm_matches_dequantized_reference(
|
||||
shape, tight_atol, max_atol, pass_rate
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
|
||||
batch, m_dim, k_dim = shape
|
||||
n_dim = k_dim
|
||||
a = torch.randn(batch * m_dim, k_dim, device=DEVICE)
|
||||
b = torch.randn(n_dim, k_dim, device=DEVICE)
|
||||
|
||||
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
|
||||
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
|
||||
out = torch._scaled_mm(
|
||||
a_fp8,
|
||||
b_fp8.t(),
|
||||
out_dtype=torch.bfloat16,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
)
|
||||
|
||||
a_dq = a_fp8.float() * scale_a.float()
|
||||
b_dq = b_fp8.float() * scale_b.float()
|
||||
ref = torch.mm(a_dq, b_dq.t())
|
||||
|
||||
assert out.shape == (batch * m_dim, n_dim)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
_assert_abs_error_budget(
|
||||
out.float(),
|
||||
ref,
|
||||
label=f"scaled_mm shape={shape}",
|
||||
tight_atol=tight_atol,
|
||||
max_atol=max_atol,
|
||||
pass_rate=pass_rate,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_fp8_scaled_mm_determinism():
|
||||
torch.manual_seed(5)
|
||||
|
||||
m_dim, k_dim, n_dim = 64, 128, 128
|
||||
a = torch.randn(m_dim, k_dim, device=DEVICE)
|
||||
b = torch.randn(n_dim, k_dim, device=DEVICE)
|
||||
a_fp8, scale_a = ref_dynamic_per_tensor_fp8_quant(a)
|
||||
b_fp8, scale_b = ref_dynamic_per_tensor_fp8_quant(b)
|
||||
|
||||
def run_gemm():
|
||||
return torch._scaled_mm(
|
||||
a_fp8,
|
||||
b_fp8.t(),
|
||||
out_dtype=torch.bfloat16,
|
||||
scale_a=scale_a,
|
||||
scale_b=scale_b,
|
||||
)
|
||||
|
||||
_assert_deterministic(run_gemm, n_runs=4)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_aiter_fp8_kv_attention_matches_bf16_reference():
|
||||
import aiter
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import cp_mha_gather_cache
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
set_random_seed(10)
|
||||
|
||||
num_q_heads, num_kv_heads = 8, 8
|
||||
head_size = 128
|
||||
query_len, kv_len = 4, 128
|
||||
block_size = 16
|
||||
num_blocks = 2048
|
||||
scale = head_size**-0.5
|
||||
|
||||
query = torch.randn(
|
||||
query_len, num_q_heads, head_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
key_cache_fp8 = torch.clamp(
|
||||
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
|
||||
min=-1.0,
|
||||
max=1.0,
|
||||
).to(FP8_DTYPE)
|
||||
value_cache_fp8 = torch.clamp(
|
||||
torch.randn(num_blocks, block_size, num_kv_heads, head_size, device=DEVICE),
|
||||
min=-1.0,
|
||||
max=1.0,
|
||||
).to(FP8_DTYPE)
|
||||
|
||||
cu_query_lens = torch.tensor([0, query_len], dtype=torch.int32, device=DEVICE)
|
||||
cu_seq_lens = torch.tensor([0, kv_len], dtype=torch.int32, device=DEVICE)
|
||||
block_tables = torch.randint(
|
||||
0,
|
||||
num_blocks,
|
||||
(1, (kv_len + block_size - 1) // block_size),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
token_to_batch = torch.zeros(kv_len, dtype=torch.int32, device=DEVICE)
|
||||
seq_starts = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
gathered_key = torch.empty(
|
||||
kv_len, num_kv_heads, head_size, dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
gathered_value = torch.empty_like(gathered_key)
|
||||
|
||||
cp_mha_gather_cache(
|
||||
key_cache=key_cache_fp8,
|
||||
value_cache=value_cache_fp8,
|
||||
key=gathered_key,
|
||||
value=gathered_value,
|
||||
block_tables=block_tables,
|
||||
k_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
|
||||
v_scales=torch.ones(1, dtype=torch.float32, device=DEVICE),
|
||||
cu_seqlens_kv=cu_seq_lens,
|
||||
token_to_batch=token_to_batch,
|
||||
seq_starts=seq_starts,
|
||||
dequant=True,
|
||||
kv_cache_layout="NHD",
|
||||
total_tokens=kv_len,
|
||||
)
|
||||
|
||||
output = torch.empty_like(query)
|
||||
aiter.flash_attn_varlen_func(
|
||||
q=query,
|
||||
k=gathered_key,
|
||||
v=gathered_value,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
cu_seqlens_k=cu_seq_lens,
|
||||
max_seqlen_q=query_len,
|
||||
max_seqlen_k=kv_len,
|
||||
min_seqlen_q=1,
|
||||
dropout_p=0.0,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
alibi_slopes=None,
|
||||
return_lse=False,
|
||||
out=output,
|
||||
)
|
||||
|
||||
ref = _ref_paged_attention(
|
||||
query=query,
|
||||
key_cache=key_cache_fp8.to(torch.bfloat16),
|
||||
value_cache=value_cache_fp8.to(torch.bfloat16),
|
||||
query_lens=[query_len],
|
||||
kv_lens=[kv_len],
|
||||
block_tables=block_tables,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
_assert_abs_error_budget(
|
||||
output.float(),
|
||||
ref.float(),
|
||||
label="aiter_fp8_kv_attention",
|
||||
tight_atol=0.15,
|
||||
max_atol=0.25,
|
||||
pass_rate=0.98,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
@pytest.mark.parametrize("shape", [(128, 256), (512, 1024), (1024, 4096)])
|
||||
def test_fp8_per_tensor_quant_matches_reference(shape):
|
||||
torch.manual_seed(0)
|
||||
|
||||
x = torch.randn(*shape, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scale = ops.scaled_fp8_quant(x)
|
||||
ref_out, ref_scale = ref_dynamic_per_tensor_fp8_quant(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
|
||||
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
|
||||
code_match_rate = (
|
||||
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
|
||||
)
|
||||
assert code_match_rate >= 0.999, (
|
||||
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
|
||||
)
|
||||
|
||||
dequant = out.float() * scale.float()
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label=f"per_tensor_quant shape={shape}",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.60,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_aiter_group_fp8_quant_roundtrip():
|
||||
from aiter import dtypes
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
|
||||
assert is_aiter_found_and_supported()
|
||||
|
||||
torch.manual_seed(0)
|
||||
|
||||
m_dim, n_dim = 64, 4096
|
||||
group_size = 128
|
||||
x = torch.randn(m_dim, n_dim, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scales = rocm_aiter_ops.group_fp8_quant(x, group_size)
|
||||
assert out.dtype == dtypes.fp8
|
||||
assert scales.dtype == torch.float32
|
||||
|
||||
scales_expanded = scales.repeat_interleave(group_size, dim=1)[:, :n_dim]
|
||||
dequant = out.float() * scales_expanded
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label="aiter_group_fp8_quant",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.30,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
|
||||
|
||||
@fp8_only
|
||||
def test_fp8_per_token_quant_matches_reference():
|
||||
torch.manual_seed(0)
|
||||
|
||||
x = torch.randn(32, 4096, dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
out, scale = ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True)
|
||||
ref_out, ref_scale = ref_dynamic_per_token_quant(x, FP8_DTYPE)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == FP8_DTYPE
|
||||
assert scale.shape == (x.shape[0], 1)
|
||||
|
||||
torch.testing.assert_close(scale, ref_scale, rtol=1e-3, atol=1e-4)
|
||||
code_match_rate = (
|
||||
(out.view(torch.uint8) == ref_out.view(torch.uint8)).float().mean().item()
|
||||
)
|
||||
assert code_match_rate >= 0.999, (
|
||||
f"FP8 codes match reference only {code_match_rate:.4%} (< 99.9%)"
|
||||
)
|
||||
|
||||
dequant = out.float() * scale.float()
|
||||
_assert_rel_error_budget(
|
||||
dequant,
|
||||
x.float(),
|
||||
label="per_token_quant",
|
||||
tight_rtol=0.06,
|
||||
max_rtol=0.30,
|
||||
pass_rate=0.99,
|
||||
max_fail_rate=1e-4,
|
||||
)
|
||||
@@ -0,0 +1,638 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm AITER FP4 and MXFP4 tests.
|
||||
|
||||
This file keeps the ROCm-specific FP4 coverage:
|
||||
- ROCm AITER FP4 env and enablement gates
|
||||
- large-shape MXFP4 wrapper checks through vLLM's public helper op
|
||||
- AITER Triton MXFP4 quant format and determinism
|
||||
- FP4 GEMM, preshuffled-scale, hardware-quant, and skinny decode
|
||||
paths
|
||||
|
||||
Generic MXFP4 wrapper/reference checks already live in
|
||||
``tests/kernels/quantization/test_quark.py``.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.utils import _assert_accurate, _assert_deterministic
|
||||
from tests.quantization.reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
LLAMA_SHAPES = [
|
||||
(64, 4096),
|
||||
(64, 11008),
|
||||
(32, 14336),
|
||||
]
|
||||
GEMM_ATOL = 0.5
|
||||
GEMM_RTOL = 0.0
|
||||
GEMM_MAX_VIOLATION_FACTOR = 2.0
|
||||
A4W4_GEMM_PASS_RATES = {
|
||||
(64, 128, 64): 0.9999,
|
||||
(128, 256, 128): 0.9999,
|
||||
(128, 4096, 4096): 0.9999,
|
||||
(256, 4096, 11008): 0.9999,
|
||||
(64, 8192, 28672): 0.99765,
|
||||
}
|
||||
SKINNY_GEMM_PASS_RATES = {
|
||||
(4096, 4096): 0.9999,
|
||||
(4096, 11008): 0.99235,
|
||||
(8192, 8192): 0.99765,
|
||||
}
|
||||
PRESHUFFLED_SHAPES = [
|
||||
(64, 4096, 8192),
|
||||
(32, 8192, 8192),
|
||||
]
|
||||
|
||||
|
||||
def _reload_envs():
|
||||
import vllm.envs as envs
|
||||
|
||||
return importlib.reload(envs)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_rocm_env_state():
|
||||
"""Restore global env + AITER flag state after every test."""
|
||||
yield
|
||||
_reload_envs()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
|
||||
def _assert_aiter_supported() -> None:
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
assert is_aiter_found_and_supported(), (
|
||||
"aiter is required on supported ROCm hardware for this test"
|
||||
)
|
||||
|
||||
|
||||
def _format_observed_rate(count: int, total: int) -> str:
|
||||
return f"{count / total:.4%} ({count}/{total})"
|
||||
|
||||
|
||||
def _format_allowed_rate(rate: float, total: int) -> str:
|
||||
allowed_count = int(rate * total)
|
||||
return f"{rate:.4%} (<= {allowed_count}/{total})"
|
||||
|
||||
|
||||
def _print_close_stats(
|
||||
label: str,
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
*,
|
||||
atol: float,
|
||||
rtol: float,
|
||||
pass_rate: float = 1.0,
|
||||
max_atol: float | None = None,
|
||||
) -> None:
|
||||
abs_diff = (actual - expected).abs().float().flatten()
|
||||
expected_abs = expected.abs().float().flatten()
|
||||
allowed = atol + rtol * expected_abs
|
||||
within = abs_diff <= allowed
|
||||
|
||||
total = abs_diff.numel()
|
||||
passed = int(within.sum().item())
|
||||
failed = total - passed
|
||||
allowed_fail_rate = 1.0 - pass_rate
|
||||
|
||||
max_abs = abs_diff.max().item()
|
||||
mean_abs = abs_diff.mean().item()
|
||||
p99_abs = torch.quantile(abs_diff, 0.99).item()
|
||||
p999_abs = torch.quantile(abs_diff, 0.999).item()
|
||||
worst_ratio = (abs_diff / allowed.clamp_min(1e-12)).max().item()
|
||||
|
||||
msg = (
|
||||
"[rocm_aiter_fp4] "
|
||||
f"{label}: "
|
||||
f"pass={passed / total:.4%} ({passed}/{total}) "
|
||||
f"fail={_format_observed_rate(failed, total)} "
|
||||
f"allowed_fail={_format_allowed_rate(allowed_fail_rate, total)} "
|
||||
f"atol={atol:g} "
|
||||
f"rtol={rtol:g} "
|
||||
)
|
||||
if max_atol is not None:
|
||||
above_max_count = int((abs_diff > max_atol).sum().item())
|
||||
msg += (
|
||||
f"abs>{max_atol:g}={_format_observed_rate(above_max_count, total)} "
|
||||
f"allowed_above_max={_format_allowed_rate(0.0, total)} "
|
||||
)
|
||||
msg += (
|
||||
f"max_abs={max_abs:.6g} "
|
||||
f"mean_abs={mean_abs:.6g} "
|
||||
f"p99_abs={p99_abs:.6g} "
|
||||
f"p999_abs={p999_abs:.6g} "
|
||||
f"worst_ratio={worst_ratio:.6g}"
|
||||
)
|
||||
print(msg)
|
||||
if failed > 0:
|
||||
warnings.warn(msg, stacklevel=2)
|
||||
|
||||
|
||||
# Env and enablement tests ------------------------------------------------
|
||||
|
||||
|
||||
def test_fp4_env_defaults():
|
||||
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"use_aiter",
|
||||
"use_fp4_asm_gemm",
|
||||
"use_fp4bmm",
|
||||
),
|
||||
[
|
||||
(True, True, True),
|
||||
(True, True, False),
|
||||
(True, False, True),
|
||||
(False, True, True),
|
||||
],
|
||||
)
|
||||
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
use_aiter,
|
||||
use_fp4_asm_gemm,
|
||||
use_fp4bmm,
|
||||
monkeypatch,
|
||||
):
|
||||
"""The ROCm FP4 AITER gates should depend only on the env toggles and the
|
||||
gfx950 hardware check.
|
||||
|
||||
Expected values are derived from the *actual* platform rather than a patched
|
||||
arch (gfx950 -> MI355, gfx942 -> MI300). A failure in this group therefore
|
||||
points straight at the hardware it ran on instead of a mocked value.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
on_gfx950_value = on_gfx950()
|
||||
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
|
||||
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv(
|
||||
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
|
||||
"1" if use_fp4_asm_gemm else "0",
|
||||
)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert (
|
||||
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
|
||||
)
|
||||
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
|
||||
|
||||
|
||||
# Large-shape MXFP4 wrapper tests -----------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
|
||||
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
|
||||
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
|
||||
reference even on larger Llama-like shapes."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
rows, cols = shape
|
||||
x = torch.randn(rows, cols, dtype=dtype)
|
||||
out = quant_dequant_mxfp4(x)
|
||||
ref = qdq_mxfp4_torch(x, "even")
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert out.dtype == dtype
|
||||
assert torch.isfinite(out).all()
|
||||
torch.testing.assert_close(out, ref, atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
def test_vllm_quant_dequant_mxfp4_is_deterministic():
|
||||
"""The public vLLM MXFP4 QDQ helper should stay bitwise deterministic."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(3)
|
||||
|
||||
x = torch.randn(64, 128, dtype=torch.bfloat16)
|
||||
|
||||
_assert_deterministic(quant_dequant_mxfp4, x, n_runs=4)
|
||||
|
||||
|
||||
# Triton MXFP4 quant tests ------------------------------------------------
|
||||
|
||||
|
||||
def test_aiter_dynamic_mxfp4_quant_output_format():
|
||||
"""dynamic_mxfp4_quant returns packed uint8 FP4 values and E8M0 uint8 scales.
|
||||
|
||||
OCP MXFP4: block_size=32, 2 FP4 E2M1 values packed per byte.
|
||||
Scale shape: (M, K // 32); one E8M0 exponent byte per 32-element block.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = 64, 256
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x)
|
||||
|
||||
# FP4 values packed 2-per-byte; shape (M, K // 2)
|
||||
assert x_fp4.shape == (M, K // 2), (
|
||||
f"Expected fp4 shape ({M}, {K // 2}), got {x_fp4.shape}"
|
||||
)
|
||||
assert x_fp4.dtype == torch.uint8
|
||||
|
||||
# One E8M0 scale byte per 32-element block; shape (M, K // 32)
|
||||
assert x_scale.shape == (M, K // 32), (
|
||||
f"Expected scale shape ({M}, {K // 32}), got {x_scale.shape}"
|
||||
)
|
||||
assert x_scale.dtype == torch.uint8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(128, 4096), # Llama-7B hidden
|
||||
(256, 11008), # Llama-7B FFN
|
||||
(64, 14336), # Llama-70B FFN
|
||||
(512, 8192), # Llama-70B hidden
|
||||
(32, 28672), # DeepSeek-style large FFN
|
||||
],
|
||||
)
|
||||
def test_aiter_dynamic_mxfp4_quant_llama_shapes(shape, dtype):
|
||||
"""MXFP4 quantization output format is correct for Llama-class weight shapes.
|
||||
|
||||
Targets B200 parity: NVIDIA nvfp4 tests parametrize over Llama shapes
|
||||
(7168, 14336, 28672...). This covers gfx950 MXFP4 quant format validation.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = shape
|
||||
x = torch.randn(M, K, dtype=dtype)
|
||||
|
||||
x_fp4, x_scale = dynamic_mxfp4_quant(x)
|
||||
|
||||
assert x_fp4.shape == (M, K // 2)
|
||||
assert x_fp4.dtype == torch.uint8
|
||||
assert x_scale.shape == (M, K // 32)
|
||||
assert x_scale.dtype == torch.uint8
|
||||
assert not torch.any(torch.isnan(x_fp4.float()))
|
||||
|
||||
|
||||
def test_aiter_dynamic_mxfp4_quant_determinism():
|
||||
"""dynamic_mxfp4_quant is bitwise deterministic across repeated runs."""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(7)
|
||||
x = torch.randn(128, 256, dtype=torch.bfloat16)
|
||||
|
||||
_assert_deterministic(dynamic_mxfp4_quant, x, n_runs=4)
|
||||
|
||||
|
||||
# gfx950 hardware FP4 GEMM tests ------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(64, 128, 64), # small square-ish
|
||||
(128, 256, 128), # medium
|
||||
(128, 4096, 4096), # Llama-7B hidden square
|
||||
(256, 4096, 11008), # Llama-7B FFN
|
||||
(64, 8192, 28672), # Llama-70B FFN
|
||||
],
|
||||
)
|
||||
def test_aiter_fp4_gemm_a4w4_accuracy(shape):
|
||||
"""AITER A4W4 FP4 GEMM output is close to matmul on dequantized weights.
|
||||
|
||||
Tests gfx950 parity with B200 test_nvfp4_scaled_mm.py: E2M1 x E2M1 GEMM
|
||||
with block-scaled quantization, multiple Llama-class shapes.
|
||||
Requires gfx950 hardware; gracefully skips elsewhere.
|
||||
"""
|
||||
atol = GEMM_ATOL
|
||||
rtol = GEMM_RTOL
|
||||
pass_rate = A4W4_GEMM_PASS_RATES[shape]
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
M, K, N = shape
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16) # weight stored as (N, K)
|
||||
|
||||
# Quantize
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
# FP4 GEMM computes A_fp4 (M, K) @ B_fp4.T (K, N) into (M, N).
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N), f"Expected ({M}, {N}), got {out.shape}"
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
# Reference: matmul on dequantized FP4 inputs (quark roundtrip)
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float())
|
||||
|
||||
# FP4 has only 3 mantissa bits; large K amplifies the accumulation error.
|
||||
# These budgets were measured on gfx950 and keep p99/p999 drift visible
|
||||
# without pretending the kernel is close to BF16 matmul everywhere.
|
||||
_print_close_stats(
|
||||
f"a4w4_gemm shape={shape}",
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
|
||||
)
|
||||
_assert_accurate(
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
PRESHUFFLED_SHAPES,
|
||||
)
|
||||
def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
"""The preshuffled FP4 GEMM path should run on tuned shapes with the same
|
||||
packed inputs vLLM uses in production.
|
||||
|
||||
This kernel only advertises a narrow tuned `(N, K)` set on gfx950. The
|
||||
important contract here is that those tuned shapes execute with the
|
||||
production-style shuffled weights and stay deterministic.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter import per_1x32_f4_quant_hip
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import (
|
||||
gemm_afp4wfp4_preshuffled_weight_scales,
|
||||
)
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(1)
|
||||
M, K, N = shape
|
||||
|
||||
assert M <= 64
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K)
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
A_q, A_s = per_1x32_f4_quant_hip(A, shuffle=M >= 32)
|
||||
|
||||
scale_rows, scale_cols = B_scale.shape
|
||||
B_scale = (
|
||||
B_scale.view(scale_rows // 32, 2, 16, scale_cols // 8, 2, 4, 1)
|
||||
.permute(0, 3, 5, 2, 4, 1, 6)
|
||||
.contiguous()
|
||||
.view(scale_rows, scale_cols)
|
||||
)
|
||||
B_fp4 = shuffle_weight(B_fp4, layout=(16, 16))
|
||||
|
||||
if M >= 32:
|
||||
A_s = A_s.contiguous().view(torch.uint8).reshape(A_s.shape[0] // 32, -1)
|
||||
else:
|
||||
A_s = A_s[:M, ...].contiguous().view(torch.uint8)
|
||||
B_scale = B_scale.contiguous().view(torch.uint8).reshape(B_scale.shape[0] // 32, -1)
|
||||
|
||||
def run_preshuffled() -> torch.Tensor:
|
||||
y = torch.empty(M, N, device="cuda", dtype=torch.bfloat16)
|
||||
return gemm_afp4wfp4_preshuffled_weight_scales(
|
||||
A_q.contiguous().view(torch.uint8),
|
||||
B_fp4.contiguous().view(torch.uint8).reshape(B_fp4.shape[0] // 16, -1),
|
||||
A_s,
|
||||
B_scale,
|
||||
torch.bfloat16,
|
||||
y,
|
||||
)
|
||||
|
||||
out = run_preshuffled()
|
||||
assert out.shape == (M, N)
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
_assert_deterministic(run_preshuffled, n_runs=3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_gemm_a4w4_determinism():
|
||||
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
|
||||
|
||||
Targets parity with B200 FP4 determinism requirements.
|
||||
Requires gfx950.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(2)
|
||||
|
||||
M, K, N = 128, 256, 128
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
def run_gemm():
|
||||
return gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
_assert_deterministic(run_gemm, n_runs=4)
|
||||
|
||||
|
||||
# gfx950 hardware FP4 dynamic quantization --------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"shape",
|
||||
[
|
||||
(128, 256),
|
||||
(256, 4096),
|
||||
(64, 14336),
|
||||
],
|
||||
)
|
||||
def test_aiter_hardware_fp4_dynamic_quant_format(shape):
|
||||
"""aiter hardware FP4 dynamic quant produces correct output format.
|
||||
|
||||
Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1).
|
||||
Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output.
|
||||
Requires gfx950.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
|
||||
from aiter import dynamic_per_group_scaled_quant_fp4
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
M, K = shape
|
||||
group_size = 32
|
||||
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
out_fp4 = torch.empty(M, K // 2, dtype=torch.uint8)
|
||||
scales = torch.empty(M, K // group_size, dtype=torch.uint8)
|
||||
|
||||
dynamic_per_group_scaled_quant_fp4(out_fp4, x, scales, group_size)
|
||||
|
||||
assert out_fp4.shape == (M, K // 2), (
|
||||
f"Shape {shape}: expected fp4 ({M}, {K // 2}), got {out_fp4.shape}"
|
||||
)
|
||||
assert scales.shape == (M, K // group_size), (
|
||||
f"Shape {shape}: expected scale ({M}, {K // group_size}), got {scales.shape}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_hardware_fp4_quant_vs_triton():
|
||||
"""Hardware FP4 quant should dequantize back to the same MXFP4 values as
|
||||
the Triton path.
|
||||
|
||||
The packed bytes are not identical on gfx950, but with
|
||||
``shuffle_scale=False`` the dequantized results match tightly.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
|
||||
from aiter import dynamic_per_group_scaled_quant_fp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(5)
|
||||
|
||||
M, K = 128, 256
|
||||
group_size = 32
|
||||
x = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
|
||||
# Hardware path
|
||||
out_hw = torch.empty(M, K // 2, dtype=torch.uint8)
|
||||
scales_hw = torch.empty(M, K // group_size, dtype=torch.uint8)
|
||||
dynamic_per_group_scaled_quant_fp4(
|
||||
out_hw, x, scales_hw, group_size, shuffle_scale=False
|
||||
)
|
||||
|
||||
# Triton path
|
||||
out_triton, scales_triton = dynamic_mxfp4_quant(x)
|
||||
|
||||
assert out_hw.shape == out_triton.shape
|
||||
assert scales_hw.shape == scales_triton.shape
|
||||
|
||||
dq_hw = dq_mxfp4_torch(out_hw, scales_hw, torch.bfloat16)
|
||||
dq_triton = dq_mxfp4_torch(out_triton, scales_triton, torch.bfloat16)
|
||||
|
||||
_print_close_stats(
|
||||
"hardware_quant_vs_triton",
|
||||
dq_hw.float(),
|
||||
dq_triton.float(),
|
||||
atol=0.25,
|
||||
rtol=0.0,
|
||||
)
|
||||
torch.testing.assert_close(dq_hw, dq_triton, atol=0.25, rtol=0.0)
|
||||
|
||||
|
||||
# ROCm skinny GEMM FP4 tests ----------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
@pytest.mark.parametrize("M", [1, 2, 4, 8]) # decode / skinny batch sizes
|
||||
@pytest.mark.parametrize("N, K", [(4096, 4096), (4096, 11008), (8192, 8192)])
|
||||
def test_aiter_fp4_gemm_skinny_shapes(M, N, K):
|
||||
"""FP4 GEMM accuracy for skinny (small-M) shapes (decode phase).
|
||||
|
||||
Skinny GEMMs (M=1..8) are the bottleneck in decode phase.
|
||||
Tests parity with B200 nvfp4_scaled_mm tests at decode batch sizes.
|
||||
Requires gfx950.
|
||||
"""
|
||||
atol = GEMM_ATOL
|
||||
rtol = GEMM_RTOL
|
||||
pass_rate = SKINNY_GEMM_PASS_RATES[(N, K)]
|
||||
_assert_aiter_supported()
|
||||
from aiter.ops.triton.gemm_afp4wfp4 import gemm_afp4wfp4
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(0)
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
|
||||
A_fp4, A_scale = dynamic_mxfp4_quant(A)
|
||||
B_fp4, B_scale = dynamic_mxfp4_quant(B)
|
||||
|
||||
out = gemm_afp4wfp4(A_fp4, B_fp4, A_scale, B_scale)
|
||||
|
||||
assert out.shape == (M, N)
|
||||
assert not torch.any(torch.isnan(out))
|
||||
|
||||
A_dq = quant_dequant_mxfp4(A)
|
||||
B_dq = quant_dequant_mxfp4(B)
|
||||
ref = torch.matmul(A_dq.float(), B_dq.t().float())
|
||||
|
||||
_print_close_stats(
|
||||
f"skinny_gemm M={M} N={N} K={K}",
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_atol=GEMM_MAX_VIOLATION_FACTOR * atol,
|
||||
)
|
||||
_assert_accurate(
|
||||
out.float(),
|
||||
ref,
|
||||
atol=atol,
|
||||
rtol=rtol,
|
||||
pass_rate=pass_rate,
|
||||
max_violation_factor=GEMM_MAX_VIOLATION_FACTOR,
|
||||
)
|
||||
@@ -36,6 +36,63 @@ ALL_OPCHECK_TEST_UTILS: tuple[str, ...] = (
|
||||
)
|
||||
|
||||
|
||||
def _assert_accurate(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
atol: float,
|
||||
rtol: float = 0.0,
|
||||
pass_rate: float = 0.99999,
|
||||
max_violation_factor: float = 3.0,
|
||||
) -> None:
|
||||
"""Check numeric accuracy with pass-rate, max-error, and mean-error bounds."""
|
||||
a = actual.detach().float().flatten()
|
||||
e = expected.detach().float().flatten()
|
||||
|
||||
abs_err = (a - e).abs()
|
||||
tol = atol + rtol * e.abs()
|
||||
|
||||
rate = (abs_err <= tol).float().mean().item()
|
||||
assert rate >= pass_rate, (
|
||||
f"Accuracy pass rate {rate:.6f} < {pass_rate} (atol={atol}, rtol={rtol})"
|
||||
)
|
||||
|
||||
max_err = abs_err.max().item()
|
||||
assert max_err <= max_violation_factor * atol, (
|
||||
f"Max absolute error {max_err:.6f} exceeds {max_violation_factor} * atol={atol}"
|
||||
)
|
||||
|
||||
mean_err = abs_err.mean().item()
|
||||
assert mean_err <= atol * 0.25, (
|
||||
f"Mean absolute error {mean_err:.6f} >= atol * 0.25 = {atol * 0.25:.6f}"
|
||||
)
|
||||
|
||||
|
||||
def _assert_deterministic(
|
||||
fn,
|
||||
*args,
|
||||
n_runs: int = 4,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Verify that repeated calls produce bitwise-identical tensor outputs."""
|
||||
|
||||
def _collect(result: Any) -> list[torch.Tensor]:
|
||||
if isinstance(result, torch.Tensor):
|
||||
return [result.detach().clone()]
|
||||
if isinstance(result, (tuple, list)):
|
||||
return [t.detach().clone() for t in result if isinstance(t, torch.Tensor)]
|
||||
raise TypeError(f"Unexpected return type {type(result)}")
|
||||
|
||||
reference = _collect(fn(*args, **kwargs))
|
||||
|
||||
for run in range(1, n_runs):
|
||||
outputs = _collect(fn(*args, **kwargs))
|
||||
for idx, (ref, out) in enumerate(zip(reference, outputs)):
|
||||
assert torch.equal(ref, out), (
|
||||
f"Run {run}: output[{idx}] differs from run 0 "
|
||||
f"(max diff = {(out.float() - ref.float()).abs().max().item():.2e})"
|
||||
)
|
||||
|
||||
|
||||
class QKVInputs(NamedTuple):
|
||||
"""
|
||||
Data structure for representing unpacked attention inputs,
|
||||
|
||||
Reference in New Issue
Block a user