[Bugfix][Quantization] Fix fp32 weight scale for mxfp4 quantization and per-expert checkpoint mapping (#51419)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-09 23:08:52 -07:00
committed by GitHub
parent b22afe45ac
commit 3b4c86e489
2 changed files with 67 additions and 0 deletions
@@ -896,6 +896,14 @@ class RoutedExperts(PluggableLayer):
break
continue
matched = True
is_per_expert_fused_w13 = (
not is_fused
and shard_id in {"w1", "w3"}
and any(
f".{fused_name}." in qual_name
for fused_name in ("gate_up_proj", "w13")
)
)
weight_name = qual_name.replace(weight_name, param_name)
param_name = weight_name.removeprefix(f"{self.layer_name}.")
param = getattr(self, param_name, None)
@@ -919,6 +927,12 @@ class RoutedExperts(PluggableLayer):
else:
experts_shard = fused_weight
start = 0
elif is_per_expert_fused_w13:
shard_index = 0 if shard_id == "w1" else 1
experts_shard = loaded_weight.chunk(2, dim=0)[
shard_index
].unsqueeze(0)
start = expert_id
else:
# loaded_weight is a single expert weight, so we add a dummy expert
# dimension to unify the loading logic with the fused case
@@ -1069,6 +1083,19 @@ class RoutedExperts(PluggableLayer):
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
]
fused_mapping.extend(
(
w13,
(
f"experts.{physical_to_logical_map[expert_id]}."
f"{gate_up}.{lora_base_layer_prefix}"
),
expert_id,
shard_id,
)
for expert_id in range(num_physical_experts)
for shard_id in ("w1", "w3")
)
per_expert_mapping = [
# (param_name, weight_name, expert_id, shard_id)
@@ -569,6 +569,38 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
activation=self.moe.activation,
)
@staticmethod
def _encode_mxfp4_weight_scale(loaded_weight: torch.Tensor) -> torch.Tensor:
if loaded_weight.dtype == torch.uint8:
return loaded_weight
if loaded_weight.dtype == torch.float8_e8m0fnu:
return loaded_weight.view(torch.uint8)
if loaded_weight.is_floating_point():
return loaded_weight.to(torch.float8_e8m0fnu).view(torch.uint8)
return loaded_weight
@staticmethod
def get_scale_weight_loader(weight_loader):
def mxfp4_weight_loader(
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
loaded_weight = Mxfp4MoEMethod._encode_mxfp4_weight_scale(loaded_weight)
return weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success=return_success,
)
return mxfp4_weight_loader
def create_weights(
self,
layer: RoutedExperts,
@@ -587,6 +619,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
layer.num_experts = num_experts
self.intermediate_size = intermediate_size_per_partition
self.hidden_size = hidden_size
weight_loader = extra_weight_attrs.pop("weight_loader")
scale_weight_loader = Mxfp4MoEMethod.get_scale_weight_loader(weight_loader)
# Fused gate_up_proj (column parallel)
w13_weight = torch.nn.Parameter(
@@ -600,6 +634,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_weight", w13_weight)
set_weight_attrs(w13_weight, extra_weight_attrs)
set_weight_attrs(w13_weight, {"weight_loader": weight_loader})
w13_weight_scale = torch.nn.Parameter(
torch.zeros(
@@ -612,6 +647,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_weight_scale", w13_weight_scale)
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
set_weight_attrs(w13_weight_scale, {"weight_loader": scale_weight_loader})
w13_weight_scale.quant_method = "block"
# down_proj (row parallel)
@@ -626,6 +662,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_weight", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
set_weight_attrs(w2_weight, {"weight_loader": weight_loader})
w2_weight_scale = torch.nn.Parameter(
torch.zeros(
@@ -638,6 +675,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_weight_scale", w2_weight_scale)
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
set_weight_attrs(w2_weight_scale, {"weight_loader": scale_weight_loader})
w2_weight_scale.quant_method = "block"
if self.moe.has_bias:
@@ -651,6 +689,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
set_weight_attrs(w13_bias, {"weight_loader": weight_loader})
w2_bias = torch.nn.Parameter(
torch.zeros(
@@ -662,6 +701,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
set_weight_attrs(w2_bias, {"weight_loader": weight_loader})
def _setup_kernel(
self,