[Kernel] Add opt-in load-time MXFP4 dequantization (#50814)

Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
yinfengLiu
2026-09-22 14:51:12 +08:00
committed by GitHub
co-authored by Shanshan Shen
parent 68363d64ec
commit 2af05512cf
3 changed files with 112 additions and 3 deletions
@@ -8,6 +8,7 @@ import torch
import torch.nn.functional as F
from torch.nn.parameter import Parameter
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
dequant_mxfp4,
@@ -97,7 +98,16 @@ class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
# Match the MXFP8 emulation path when explicitly enabled: dequantize
# once at load time so inference uses a plain high-precision linear.
# Otherwise retain packed weights and dequantize per invocation.
if envs.VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD:
weight = dequant_mxfp4(layer.weight, weight_scale, torch.bfloat16)
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
layer.weight_scale = weight_scale
def apply_weights(
self,
@@ -105,6 +115,12 @@ class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
qdq_x = self.quant_dequant_func(x)
# Packed MXFP4 weights use one byte for two values. A weight with
# elements wider than one byte was already dequantized at load time.
if layer.weight.element_size() >= 2:
return F.linear(qdq_x, layer.weight.to(x.dtype), bias)
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
return F.linear(qdq_x, dq_w, bias)