[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-14 15:49:37 +08:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang
parent e1726d9142
commit c9bfa574ce
16 changed files with 1353 additions and 148 deletions
@@ -127,6 +127,7 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
replace_parameter(layer, "weight_scale", weight_scale.data)
self.kernel.process_weights_after_loading(layer)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -295,7 +296,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
self._zero_padding(layer)
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
self._zero_padding(layer, w13_weight, w2_weight)
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
layer._already_called_process_weights_after_loading = True
@@ -304,8 +306,8 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
layer.w13_input_scale = None
layer.w2_input_scale = None
w13, w13_scale = _quantize_mxfp4_moe_weight(layer.w13_weight)
w2, w2_scale = _quantize_mxfp4_moe_weight(layer.w2_weight)
w13, w13_scale = _quantize_mxfp4_moe_weight(w13_weight)
w2, w2_scale = _quantize_mxfp4_moe_weight(w2_weight)
self._setup_kernel(
layer,
@@ -316,5 +318,6 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase):
getattr(layer, "w13_bias", None),
getattr(layer, "w2_bias", None),
)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True