[ROCm][Quark][6/N] Use MXFP4 linear kernel abstraction for aiter backend (#49348)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
73af7a362a
commit
7aea73d83d
@@ -0,0 +1,5 @@
|
||||
model_name: "amd-quark/Qwen3-1.7B-MXFP4"
|
||||
accuracy_threshold: 0.27
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 4096"
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MXFP4 linear kernel selection logic (CPU-only)
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
AiterMxfp4LinearKernel,
|
||||
MxFp4LinearKernel,
|
||||
MxFp4LinearLayerConfig,
|
||||
init_mxfp4_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.platforms import PlatformEnum
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_can_implement_is_abstract():
|
||||
"""Test that can_implement()/is_supported() are properly defined."""
|
||||
assert hasattr(MxFp4LinearKernel, "can_implement")
|
||||
assert hasattr(MxFp4LinearKernel, "is_supported")
|
||||
|
||||
|
||||
def test_aiter_kernel_is_supported_requires_native_mx_support():
|
||||
"""AiterMxfp4LinearKernel must not be selected on platforms without
|
||||
native MX compute, even if AITER itself is importable."""
|
||||
with patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform.supports_mx",
|
||||
return_value=False,
|
||||
):
|
||||
is_supported, reason = AiterMxfp4LinearKernel.is_supported()
|
||||
assert not is_supported
|
||||
assert reason
|
||||
|
||||
|
||||
class OOTMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_dispatches_to_registered_kernel(platform_mock):
|
||||
"""init_mxfp4_linear_kernel should select a registered kernel that
|
||||
reports itself as supported, and construct it with a fresh config."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTMxFp4LinearKernel, PlatformEnum.OOT, "mxfp4")
|
||||
|
||||
kernel = init_mxfp4_linear_kernel()
|
||||
|
||||
assert isinstance(kernel, OOTMxFp4LinearKernel)
|
||||
assert kernel.config == MxFp4LinearLayerConfig()
|
||||
|
||||
|
||||
class UnsupportedMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
return False, "never supported"
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
pass
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_raises_when_no_kernel_matches(platform_mock):
|
||||
platform_mock._enum = PlatformEnum.UNSPECIFIED
|
||||
register_linear_kernel(
|
||||
UnsupportedMxFp4LinearKernel, PlatformEnum.UNSPECIFIED, "mxfp4"
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Failed to find a kernel"):
|
||||
init_mxfp4_linear_kernel()
|
||||
|
||||
|
||||
@patch("vllm.model_executor.kernels.linear.mxfp4.aiter.is_aiter_found_and_supported")
|
||||
@patch("vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform")
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_raises_on_rocm_without_aiter(
|
||||
linear_platform_mock, aiter_platform_mock, is_aiter_found_and_supported_mock
|
||||
):
|
||||
"""On ROCm, the only registered MXFP4 linear kernel is AITER-based.
|
||||
If AITER is not found/supported, no kernel should be selected."""
|
||||
linear_platform_mock._enum = PlatformEnum.ROCM
|
||||
aiter_platform_mock.supports_mx.return_value = True
|
||||
is_aiter_found_and_supported_mock.return_value = False
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="(?s)Failed to find a kernel.*"
|
||||
"AITER not found or not supported on the current platform",
|
||||
):
|
||||
init_mxfp4_linear_kernel()
|
||||
@@ -17,6 +17,7 @@ import pytest
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
|
||||
QuarkLinearMethod,
|
||||
QuarkW8A8Fp8,
|
||||
@@ -26,6 +27,9 @@ from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E
|
||||
QuarkW4A8Fp8MoEMethod,
|
||||
QuarkW8A8Int8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
is_layer_skipped,
|
||||
)
|
||||
@@ -52,6 +56,8 @@ QUARK_MXFP4_AVAILABLE = find_spec("quark") is not None and version.parse(
|
||||
importlib.metadata.version("amd-quark")
|
||||
) >= version.parse(QUARK_MXFP4_MIN_VERSION)
|
||||
|
||||
AITER_AVAILABLE = is_aiter_found_and_supported()
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
if QUARK_MXFP4_AVAILABLE:
|
||||
@@ -487,6 +493,42 @@ def test_mxfp4_dequant_kernel_match_quark(
|
||||
assert torch.equal(out_hip, out_torch)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not QUARK_MXFP4_AVAILABLE,
|
||||
reason=f"amd-quark>={QUARK_MXFP4_MIN_VERSION} is not available",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not AITER_AVAILABLE,
|
||||
reason="AITER is not found or not supported on the current platform",
|
||||
)
|
||||
@pytest.mark.parametrize("float_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("scalings", [[2.3, 0.03, 7.3, 0.1, 0.004, 17.3, 1e4, 1e-4]])
|
||||
def test_mxfp4_dynamic_quant_match_quark(
|
||||
float_dtype: torch.dtype, scalings: list[float]
|
||||
):
|
||||
"""`AiterMxfp4LinearKernel` quantizes weights dynamically through AITER's
|
||||
`dynamic_mxfp4_quant`, while the emulation path quantizes/dequantizes
|
||||
through Quark's `qdq_mxfp4`. Check that both agree on the same input.
|
||||
"""
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
torch.manual_seed(0)
|
||||
|
||||
hidden_size = 32 * 64
|
||||
inp = (torch.rand(48, hidden_size, dtype=float_dtype, device=DEVICE_TYPE) - 0.5) * 2
|
||||
for i in range(hidden_size // 32):
|
||||
inp[:, i * 32 : (i + 1) * 32] = (
|
||||
inp[:, i * 32 : (i + 1) * 32] * scalings[i % len(scalings)]
|
||||
)
|
||||
|
||||
x_q, x_s = dynamic_mxfp4_quant(inp)
|
||||
out_dynamic_quant = dq_mxfp4_torch(x_q, x_s, float_dtype)
|
||||
|
||||
out_quark_qdq = quant_dequant_mxfp4(inp)
|
||||
|
||||
assert torch.equal(out_dynamic_quant, out_quark_qdq)
|
||||
|
||||
|
||||
# Unit tests for ``is_layer_skipped`` fused-name handling.
|
||||
|
||||
FUSED_MAPPING = {
|
||||
|
||||
Reference in New Issue
Block a user