[Kernel] Add opt-in load-time MXFP4 dequantization (#50814)
Signed-off-by: Liuyinfeng01 <yinfeliu@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Shanshan Shen
parent
68363d64ec
commit
2af05512cf
@@ -8,6 +8,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
dequant_mxfp4,
|
||||
@@ -97,7 +98,16 @@ class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
|
||||
weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
|
||||
|
||||
# Match the MXFP8 emulation path when explicitly enabled: dequantize
|
||||
# once at load time so inference uses a plain high-precision linear.
|
||||
# Otherwise retain packed weights and dequantize per invocation.
|
||||
if envs.VLLM_MXFP4_EMULATION_DEQUANT_AT_LOAD:
|
||||
weight = dequant_mxfp4(layer.weight, weight_scale, torch.bfloat16)
|
||||
layer.weight = Parameter(weight.contiguous(), requires_grad=False)
|
||||
|
||||
layer.weight_scale = weight_scale
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -105,6 +115,12 @@ class EmulationMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
|
||||
qdq_x = self.quant_dequant_func(x)
|
||||
|
||||
# Packed MXFP4 weights use one byte for two values. A weight with
|
||||
# elements wider than one byte was already dequantized at load time.
|
||||
if layer.weight.element_size() >= 2:
|
||||
return F.linear(qdq_x, layer.weight.to(x.dtype), bias)
|
||||
|
||||
dq_w = dequant_mxfp4(layer.weight, layer.weight_scale, x.dtype)
|
||||
return F.linear(qdq_x, dq_w, bias)
|
||||
|
||||
Reference in New Issue
Block a user