[Bugfix][Quantization] Fix fp32 weight scale for mxfp4 quantization and per-expert checkpoint mapping (#51419)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -569,6 +569,38 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
activation=self.moe.activation,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _encode_mxfp4_weight_scale(loaded_weight: torch.Tensor) -> torch.Tensor:
|
||||
if loaded_weight.dtype == torch.uint8:
|
||||
return loaded_weight
|
||||
if loaded_weight.dtype == torch.float8_e8m0fnu:
|
||||
return loaded_weight.view(torch.uint8)
|
||||
if loaded_weight.is_floating_point():
|
||||
return loaded_weight.to(torch.float8_e8m0fnu).view(torch.uint8)
|
||||
return loaded_weight
|
||||
|
||||
@staticmethod
|
||||
def get_scale_weight_loader(weight_loader):
|
||||
def mxfp4_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
loaded_weight = Mxfp4MoEMethod._encode_mxfp4_weight_scale(loaded_weight)
|
||||
return weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success=return_success,
|
||||
)
|
||||
|
||||
return mxfp4_weight_loader
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -587,6 +619,8 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
layer.num_experts = num_experts
|
||||
self.intermediate_size = intermediate_size_per_partition
|
||||
self.hidden_size = hidden_size
|
||||
weight_loader = extra_weight_attrs.pop("weight_loader")
|
||||
scale_weight_loader = Mxfp4MoEMethod.get_scale_weight_loader(weight_loader)
|
||||
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_weight = torch.nn.Parameter(
|
||||
@@ -600,6 +634,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_weight", w13_weight)
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
set_weight_attrs(w13_weight, {"weight_loader": weight_loader})
|
||||
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -612,6 +647,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_weight_scale", w13_weight_scale)
|
||||
set_weight_attrs(w13_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w13_weight_scale, {"weight_loader": scale_weight_loader})
|
||||
w13_weight_scale.quant_method = "block"
|
||||
|
||||
# down_proj (row parallel)
|
||||
@@ -626,6 +662,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight, {"weight_loader": weight_loader})
|
||||
|
||||
w2_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -638,6 +675,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_weight_scale", w2_weight_scale)
|
||||
set_weight_attrs(w2_weight_scale, extra_weight_attrs)
|
||||
set_weight_attrs(w2_weight_scale, {"weight_loader": scale_weight_loader})
|
||||
w2_weight_scale.quant_method = "block"
|
||||
|
||||
if self.moe.has_bias:
|
||||
@@ -651,6 +689,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
set_weight_attrs(w13_bias, {"weight_loader": weight_loader})
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
@@ -662,6 +701,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
set_weight_attrs(w2_bias, {"weight_loader": weight_loader})
|
||||
|
||||
def _setup_kernel(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user