[MoE] Free unused MXFP4 scales in OAI Triton Backend (#46549)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-06-24 00:06:41 -07:00
committed by GitHub
parent 9d6fdc2901
commit 96de8bb389
2 changed files with 20 additions and 4 deletions
@@ -393,16 +393,19 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
def get_fused_moe_quant_config(
self, layer: RoutedExperts
) -> FusedMoEQuantConfig | None:
w1_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
w1_bias = getattr(layer, "w13_bias", None)
w2_bias = getattr(layer, "w2_bias", None)
if self.mxfp4_backend in TRITON_BACKENDS:
# TRITON backends free w13/w2_weight_scale after swizzling; the
# swizzled scales live inside the precision configs instead.
assert self.w13_precision_config is not None
assert self.w2_precision_config is not None
w1_scale = self.w13_precision_config
w2_scale = self.w2_precision_config
else:
w1_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,
@@ -738,17 +741,20 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self,
layer: RoutedExperts,
) -> FusedMoEQuantConfig | None:
w1_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
w1_bias = getattr(layer, "w13_bias", None)
w2_bias = getattr(layer, "w2_bias", None)
swiglu_limit = getattr(layer, "swiglu_limit", None)
if self.mxfp4_backend in TRITON_BACKENDS:
# TRITON backends free w13/w2_weight_scale after swizzling; the
# swizzled scales live inside the precision configs instead.
assert self.w13_precision_config is not None
assert self.w2_precision_config is not None
w1_scale = self.w13_precision_config
w2_scale = self.w2_precision_config
else:
w1_scale = layer.w13_weight_scale
w2_scale = layer.w2_weight_scale
return make_mxfp4_moe_quant_config(
mxfp4_backend=self.mxfp4_backend,