[MoE] Free unused MXFP4 scales in OAI Triton Backend (#46549)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
@@ -393,16 +393,19 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: RoutedExperts
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w1_bias = getattr(layer, "w13_bias", None)
|
||||
w2_bias = getattr(layer, "w2_bias", None)
|
||||
|
||||
if self.mxfp4_backend in TRITON_BACKENDS:
|
||||
# TRITON backends free w13/w2_weight_scale after swizzling; the
|
||||
# swizzled scales live inside the precision configs instead.
|
||||
assert self.w13_precision_config is not None
|
||||
assert self.w2_precision_config is not None
|
||||
w1_scale = self.w13_precision_config
|
||||
w2_scale = self.w2_precision_config
|
||||
else:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
@@ -738,17 +741,20 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
w1_bias = getattr(layer, "w13_bias", None)
|
||||
w2_bias = getattr(layer, "w2_bias", None)
|
||||
swiglu_limit = getattr(layer, "swiglu_limit", None)
|
||||
|
||||
if self.mxfp4_backend in TRITON_BACKENDS:
|
||||
# TRITON backends free w13/w2_weight_scale after swizzling; the
|
||||
# swizzled scales live inside the precision configs instead.
|
||||
assert self.w13_precision_config is not None
|
||||
assert self.w2_precision_config is not None
|
||||
w1_scale = self.w13_precision_config
|
||||
w2_scale = self.w2_precision_config
|
||||
else:
|
||||
w1_scale = layer.w13_weight_scale
|
||||
w2_scale = layer.w2_weight_scale
|
||||
|
||||
return make_mxfp4_moe_quant_config(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
|
||||
Reference in New Issue
Block a user