[Kernel] Add opt-in load-time MXFP4 dequantization (#50814)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
68363d64ec
commit
2af05512cf
@@ -5,11 +5,12 @@
|
||||
Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`.
|
||||
"""
|
||||
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
AiterMxfp4LinearKernel,
|
||||
EmulationMxfp4LinearKernel,
|
||||
@@ -46,6 +47,19 @@ _TRUE_W4A4_KERNELS = [
|
||||
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel]
|
||||
|
||||
|
||||
def _make_emulation_layer():
|
||||
layer = torch.nn.Module()
|
||||
layer.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(torch.ones((2, 2), dtype=torch.uint8), False),
|
||||
)
|
||||
layer.register_parameter(
|
||||
"weight_scale",
|
||||
torch.nn.Parameter(torch.ones((2, 1), dtype=torch.uint8), False),
|
||||
)
|
||||
return layer
|
||||
|
||||
|
||||
def test_can_implement_is_abstract():
|
||||
"""Test that can_implement()/is_supported() are properly defined."""
|
||||
assert hasattr(MxFp4LinearKernel, "can_implement")
|
||||
@@ -155,6 +169,77 @@ def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
|
||||
assert kernel.quant_dequant_func is quant_dequant_mxfp4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_emulation_kernel_dequantizes_at_load_and_keeps_activation_qdq(
|
||||
dtype, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
kernel = EmulationMxfp4LinearKernel(
|
||||
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
)
|
||||
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
|
||||
layer = _make_emulation_layer()
|
||||
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
|
||||
x = torch.tensor([[2.0, 4.0]], dtype=dtype)
|
||||
|
||||
with (
|
||||
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", True),
|
||||
patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
|
||||
return_value=dequantized,
|
||||
) as dequant,
|
||||
):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
actual = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = torch.nn.functional.linear(x * 0.5, dequantized.to(dtype))
|
||||
torch.testing.assert_close(actual, expected)
|
||||
dequant.assert_called_once()
|
||||
kernel.quant_dequant_func.assert_called_once_with(x)
|
||||
assert layer.weight.dtype == torch.bfloat16
|
||||
assert layer.weight.device == dequantized.device
|
||||
assert not layer.weight.requires_grad
|
||||
assert layer.weight_scale is not None
|
||||
assert not layer.weight_scale.requires_grad
|
||||
|
||||
|
||||
def test_emulation_kernel_opt_out_dequantizes_per_invocation(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.has_quark",
|
||||
lambda: True,
|
||||
)
|
||||
kernel = EmulationMxfp4LinearKernel(
|
||||
MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)
|
||||
)
|
||||
kernel.quant_dequant_func = MagicMock(side_effect=lambda x: x * 0.5)
|
||||
layer = _make_emulation_layer()
|
||||
dequantized = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.bfloat16)
|
||||
x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
patch.object(envs, "VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD", False),
|
||||
patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.emulation.dequant_mxfp4",
|
||||
return_value=dequantized,
|
||||
) as dequant,
|
||||
):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
dequant.assert_not_called()
|
||||
first = kernel.apply_weights(layer, x)
|
||||
second = kernel.apply_weights(layer, x)
|
||||
|
||||
expected = torch.nn.functional.linear(x * 0.5, dequantized)
|
||||
torch.testing.assert_close(first, expected)
|
||||
torch.testing.assert_close(second, expected)
|
||||
assert dequant.call_count == 2
|
||||
assert kernel.quant_dequant_func.call_count == 2
|
||||
assert layer.weight.dtype == torch.uint8
|
||||
assert layer.weight_scale is not None
|
||||
|
||||
|
||||
def test_aiter_kernel_is_supported_requires_native_mx_support():
|
||||
"""AiterMxfp4LinearKernel must not be selected on platforms without
|
||||
native MX compute, even if AITER itself is importable."""
|
||||
|
||||
Reference in New Issue
Block a user