[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
parent
e1726d9142
commit
c9bfa574ce
@@ -69,6 +69,9 @@ from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
validate_fp8_block_shape_moe,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_nvfp4_moe_weights,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
@@ -2110,6 +2113,22 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
layer.register_parameter("w2_input_scale_2", w2_input_scale_2)
|
||||
set_weight_attrs(w2_input_scale_2, extra_weight_attrs)
|
||||
|
||||
def dequantize_weight(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Dequantize Quark NVFP4 expert weights for requantization."""
|
||||
return dequantize_nvfp4_moe_weights(
|
||||
layer.w13_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w13_weight_scale_2,
|
||||
layer.w2_weight,
|
||||
layer.w2_weight_scale,
|
||||
layer.w2_weight_scale_2,
|
||||
layer.params_dtype,
|
||||
group_size=self.group_size,
|
||||
invert_global_scales=False,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
"""Convert NVFP4 MoE weights into kernel format and setup the kernel."""
|
||||
# Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
|
||||
|
||||
Reference in New Issue
Block a user