[Bugfix][Quantization] Fix fp32 weight scale for mxfp4 quantization and per-expert checkpoint mapping (#51419)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-09 23:08:52 -07:00
committed by GitHub
parent b22afe45ac
commit 3b4c86e489
2 changed files with 67 additions and 0 deletions
@@ -569,6 +569,38 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
activation=self.moe.activation,
)
@staticmethod
def _encode_mxfp4_weight_scale(loaded_weight: torch.Tensor) -> torch.Tensor:
if loaded_weight.dtype == torch.uint8:
return loaded_weight
if loaded_weight.dtype == torch.float8_e8m0fnu:
return loaded_weight.view(torch.uint8)
if loaded_weight.is_floating_point():
return loaded_weight.to(torch.float8_e8m0fnu).view(torch.uint8)
return loaded_weight
@staticmethod
def get_scale_weight_loader(weight_loader):
def mxfp4_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
loaded_weight = Mxfp4MoEMethod._encode_mxfp4_weight_scale(loaded_weight)
return weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success=return_success,
)
return mxfp4_weight_loader
def create_weights(
self,
layer: RoutedExperts,
@@ -587,6 +619,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer.num_experts = num_experts
self.intermediate_size = intermediate_size_per_partition
self.hidden_size = hidden_size
weight_loader = extra_weight_attrs.pop("weight_loader")
scale_weight_loader = Mxfp4MoEMethod.get_scale_weight_loader(weight_loader)
# Fused gate_up_proj (column parallel)
w13_weight = torch.nn.Parameter(
@@ -600,6 +634,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
set_weight_attrs(w13_weight, {"weight_loader": weight_loader})
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
@@ -612,6 +647,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
set_weight_attrs(w13_weight_scale, {"weight_loader": scale_weight_loader})
w13_weight_scale.quant_method = "block"
# down_proj (row parallel)
@@ -626,6 +662,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
set_weight_attrs(w2_weight, {"weight_loader": weight_loader})
w2_weight_scale = torch.nn.Parameter(
torch.zeros(
@@ -638,6 +675,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
set_weight_attrs(w2_weight_scale, {"weight_loader": scale_weight_loader})
w2_weight_scale.quant_method = "block"
if self.moe.has_bias:
@@ -651,6 +689,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
set_weight_attrs(w13_bias, {"weight_loader": weight_loader})
w2_bias = torch.nn.Parameter(
torch.zeros(
@@ -662,6 +701,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
set_weight_attrs(w2_bias, {"weight_loader": weight_loader})
def _setup_kernel(
self,