[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-14 15:49:37 +08:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang
parent e1726d9142
commit c9bfa574ce
16 changed files with 1353 additions and 148 deletions
@@ -81,6 +81,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_VALUE_DTYPE,
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
dequantize_nvfp4_moe_weights,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
QuantKey,
@@ -889,6 +892,22 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
"""FP4 variants use 'weight_scale_2' pattern for per-tensor weight scales."""
return True
def dequantize_weight(
self, layer: torch.nn.Module
) -> tuple[torch.Tensor, torch.Tensor]:
"""Materialize serialized ModelOpt NVFP4 expert weights."""
return dequantize_nvfp4_moe_weights(
layer.w13_weight,
layer.w13_weight_scale,
layer.w13_weight_scale_2,
layer.w2_weight,
layer.w2_weight_scale,
layer.w2_weight_scale_2,
layer.params_dtype,
group_size=self.quant_config.group_size,
invert_global_scales=False,
)
def create_weights(
self,
layer: RoutedExperts,