[Bugfix][Quantization] Fix fp32 weight scale for mxfp4 quantization and per-expert checkpoint mapping (#51419)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -896,6 +896,14 @@ class RoutedExperts(PluggableLayer):
|
||||
break
|
||||
continue
|
||||
matched = True
|
||||
is_per_expert_fused_w13 = (
|
||||
not is_fused
|
||||
and shard_id in {"w1", "w3"}
|
||||
and any(
|
||||
f".{fused_name}." in qual_name
|
||||
for fused_name in ("gate_up_proj", "w13")
|
||||
)
|
||||
)
|
||||
weight_name = qual_name.replace(weight_name, param_name)
|
||||
param_name = weight_name.removeprefix(f"{self.layer_name}.")
|
||||
param = getattr(self, param_name, None)
|
||||
@@ -919,6 +927,12 @@ class RoutedExperts(PluggableLayer):
|
||||
else:
|
||||
experts_shard = fused_weight
|
||||
start = 0
|
||||
elif is_per_expert_fused_w13:
|
||||
shard_index = 0 if shard_id == "w1" else 1
|
||||
experts_shard = loaded_weight.chunk(2, dim=0)[
|
||||
shard_index
|
||||
].unsqueeze(0)
|
||||
start = expert_id
|
||||
else:
|
||||
# loaded_weight is a single expert weight, so we add a dummy expert
|
||||
# dimension to unify the loading logic with the fused case
|
||||
@@ -1069,6 +1083,19 @@ class RoutedExperts(PluggableLayer):
|
||||
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
|
||||
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
|
||||
]
|
||||
fused_mapping.extend(
|
||||
(
|
||||
w13,
|
||||
(
|
||||
f"experts.{physical_to_logical_map[expert_id]}."
|
||||
f"{gate_up}.{lora_base_layer_prefix}"
|
||||
),
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
for expert_id in range(num_physical_experts)
|
||||
for shard_id in ("w1", "w3")
|
||||
)
|
||||
|
||||
per_expert_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
|
||||
@@ -569,6 +569,38 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
activation=self.moe.activation,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _encode_mxfp4_weight_scale(loaded_weight: torch.Tensor) -> torch.Tensor:
|
||||
if loaded_weight.dtype == torch.uint8:
|
||||
return loaded_weight
|
||||
if loaded_weight.dtype == torch.float8_e8m0fnu:
|
||||
return loaded_weight.view(torch.uint8)
|
||||
if loaded_weight.is_floating_point():
|
||||
return loaded_weight.to(torch.float8_e8m0fnu).view(torch.uint8)
|
||||
return loaded_weight
|
||||
|
||||
@staticmethod
|
||||
def get_scale_weight_loader(weight_loader):
|
||||
def mxfp4_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
loaded_weight = Mxfp4MoEMethod._encode_mxfp4_weight_scale(loaded_weight)
|
||||
return weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=return_success,
|
||||
)
|
||||
|
||||
return mxfp4_weight_loader
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -587,6 +619,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer.num_experts = num_experts
|
||||
self.intermediate_size = intermediate_size_per_partition
|
||||
self.hidden_size = hidden_size
|
||||
weight_loader = extra_weight_attrs.pop("weight_loader")
|
||||
scale_weight_loader = Mxfp4MoEMethod.get_scale_weight_loader(weight_loader)
|
||||
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_weight = torch.nn.Parameter(
|
||||
@@ -600,6 +634,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
set_weight_attrs(w13_weight, {"weight_loader": weight_loader})
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -612,6 +647,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w13_weight_scale, {"weight_loader": scale_weight_loader})
|
||||
w13_weight_scale.quant_method = "block"
|
||||
|
||||
# down_proj (row parallel)
|
||||
@@ -626,6 +662,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight, {"weight_loader": weight_loader})
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -638,6 +675,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight_scale, {"weight_loader": scale_weight_loader})
|
||||
w2_weight_scale.quant_method = "block"
|
||||
|
||||
if self.moe.has_bias:
|
||||
@@ -651,6 +689,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
set_weight_attrs(w13_bias, {"weight_loader": weight_loader})
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -662,6 +701,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
set_weight_attrs(w2_bias, {"weight_loader": weight_loader})
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user