[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-14 15:49:37 +08:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang
parent e1726d9142
commit c9bfa574ce
16 changed files with 1353 additions and 148 deletions
@@ -84,11 +84,14 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
def dequantize_weight(
self, layer: nn.Module
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
"dequantize_weight, so re-quantization from this checkpoint is "
"not supported. Please open an issue."
)
@@ -302,6 +305,9 @@ def resolve_quant_method(
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
@@ -328,11 +334,6 @@ def resolve_quant_method(
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
@@ -341,7 +342,7 @@ def resolve_quant_method(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
assert isinstance(online_quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.