[Kernel] Add opt-in load-time MXFP4 dequantization (#50814)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
yinfengLiu
2026-09-22 14:51:12 +08:00
committed by GitHub
co-authored by Shanshan Shen
parent 68363d64ec
commit 2af05512cf
3 changed files with 112 additions and 3 deletions
@@ -5,11 +5,12 @@
Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`.
"""
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
import torch
import vllm.envs as envs
from vllm.model_executor.kernels.linear import (
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
@@ -46,6 +47,19 @@ _TRUE_W4A4_KERNELS = [
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel]
def _make_emulation_layer():
layer = torch.nn.Module()
layer.register_parameter(
"weight",
torch.nn.Parameter(torch.ones((2, 2), dtype=torch.uint8), False),
)
layer.register_parameter(
"weight_scale",
torch.nn.Parameter(torch.ones((2, 1), dtype=torch.uint8), False),
)
return layer
def test_can_implement_is_abstract():
"""Test that can_implement()/is_supported() are properly defined."""
assert hasattr(MxFp4LinearKernel, "can_implement")
@@ -155,6 +169,77 @@ def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
assert kernel.quant_dequant_func is quant_dequant_mxfp4
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_emulation_kernel_dequantizes_at_load_and_keeps_activation_qdq(
dtype, monkeypatch
):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
kernel = EmulationMxfp4LinearKernel(
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
)
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
layer = _make_emulation_layer()
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
x = torch.tensor([[2.0, 4.0]], dtype=dtype)
with (
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", True),
patch(
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
return_value=dequantized,
) as dequant,
):
kernel.process_weights_after_loading(layer)
actual = kernel.apply_weights(layer, x)
expected = torch.nn.functional.linear(x * 0.5, dequantized.to(dtype))
torch.testing.assert_close(actual, expected)
dequant.assert_called_once()
kernel.quant_dequant_func.assert_called_once_with(x)
assert layer.weight.dtype == torch.bfloat16
assert layer.weight.device == dequantized.device
assert not layer.weight.requires_grad
assert layer.weight_scale is not None
assert not layer.weight_scale.requires_grad
def test_emulation_kernel_opt_out_dequantizes_per_invocation(monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
lambda: True,
)
kernel = EmulationMxfp4LinearKernel(
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
)
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
layer = _make_emulation_layer()
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16)
with (
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", False),
patch(
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
return_value=dequantized,
) as dequant,
):
kernel.process_weights_after_loading(layer)
dequant.assert_not_called()
first = kernel.apply_weights(layer, x)
second = kernel.apply_weights(layer, x)
expected = torch.nn.functional.linear(x * 0.5, dequantized)
torch.testing.assert_close(first, expected)
torch.testing.assert_close(second, expected)
assert dequant.call_count == 2
assert kernel.quant_dequant_func.call_count == 2
assert layer.weight.dtype == torch.uint8
assert layer.weight_scale is not None
def test_aiter_kernel_is_supported_requires_native_mx_support():
"""AiterMxfp4LinearKernel must not be selected on platforms without
native MX compute, even if AITER itself is importable."""