[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-14 15:49:37 +08:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang
parent e1726d9142
commit c9bfa574ce
16 changed files with 1353 additions and 148 deletions
+30 -10
View File
@@ -516,6 +516,20 @@ class ColumnParallelLinear(LinearBase):
self._maybe_allow_fp8_block_shape_mismatch()
self.gather_output = gather_output
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
quant_method = self.quant_method
if (
isinstance(quant_method, OnlineLinearBase)
and quant_method.requantization_source is not None
):
quant_method = quant_method.requantization_source
weight_loader = (
self.weight_loader_v2
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
)
self.quant_method.create_weights(
layer=self,
input_size_per_partition=self.input_size_per_partition,
@@ -523,11 +537,7 @@ class ColumnParallelLinear(LinearBase):
input_size=self.input_size,
output_size=self.output_size,
params_dtype=self.params_dtype,
weight_loader=(
self.weight_loader_v2
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
),
weight_loader=weight_loader,
)
if bias:
@@ -1697,6 +1707,20 @@ class RowParallelLinear(LinearBase):
self.input_is_parallel = input_is_parallel
self.reduce_results = reduce_results
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
quant_method = self.quant_method
if (
isinstance(quant_method, OnlineLinearBase)
and quant_method.requantization_source is not None
):
quant_method = quant_method.requantization_source
weight_loader = (
self.weight_loader_v2
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
)
self.quant_method.create_weights(
layer=self,
input_size_per_partition=self.input_size_per_partition,
@@ -1704,11 +1728,7 @@ class RowParallelLinear(LinearBase):
input_size=self.input_size,
output_size=self.output_size,
params_dtype=self.params_dtype,
weight_loader=(
self.weight_loader_v2
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
else self.weight_loader
),
weight_loader=weight_loader,
)
if not reduce_results and (bias and not skip_bias_add):
raise ValueError(