[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
parent
e1726d9142
commit
c9bfa574ce
@@ -516,6 +516,20 @@ class ColumnParallelLinear(LinearBase):
|
||||
self._maybe_allow_fp8_block_shape_mismatch()
|
||||
self.gather_output = gather_output
|
||||
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
quant_method = self.quant_method
|
||||
if (
|
||||
isinstance(quant_method, OnlineLinearBase)
|
||||
and quant_method.requantization_source is not None
|
||||
):
|
||||
quant_method = quant_method.requantization_source
|
||||
weight_loader = (
|
||||
self.weight_loader_v2
|
||||
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
)
|
||||
|
||||
self.quant_method.create_weights(
|
||||
layer=self,
|
||||
input_size_per_partition=self.input_size_per_partition,
|
||||
@@ -523,11 +537,7 @@ class ColumnParallelLinear(LinearBase):
|
||||
input_size=self.input_size,
|
||||
output_size=self.output_size,
|
||||
params_dtype=self.params_dtype,
|
||||
weight_loader=(
|
||||
self.weight_loader_v2
|
||||
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
|
||||
if bias:
|
||||
@@ -1697,6 +1707,20 @@ class RowParallelLinear(LinearBase):
|
||||
self.input_is_parallel = input_is_parallel
|
||||
self.reduce_results = reduce_results
|
||||
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
quant_method = self.quant_method
|
||||
if (
|
||||
isinstance(quant_method, OnlineLinearBase)
|
||||
and quant_method.requantization_source is not None
|
||||
):
|
||||
quant_method = quant_method.requantization_source
|
||||
weight_loader = (
|
||||
self.weight_loader_v2
|
||||
if quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
)
|
||||
|
||||
self.quant_method.create_weights(
|
||||
layer=self,
|
||||
input_size_per_partition=self.input_size_per_partition,
|
||||
@@ -1704,11 +1728,7 @@ class RowParallelLinear(LinearBase):
|
||||
input_size=self.input_size,
|
||||
output_size=self.output_size,
|
||||
params_dtype=self.params_dtype,
|
||||
weight_loader=(
|
||||
self.weight_loader_v2
|
||||
if self.quant_method.__class__.__name__ in WEIGHT_LOADER_V2_SUPPORTED
|
||||
else self.weight_loader
|
||||
),
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
if not reduce_results and (bias and not skip_bias_add):
|
||||
raise ValueError(
|
||||
|
||||
Reference in New Issue
Block a user