[Online quantization] Extend re-quantization API to MOE (showcase NVFP4 modelopt/CT/quark to any online MOE method) (#60276)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
parent
e1726d9142
commit
c9bfa574ce
@@ -34,7 +34,6 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
@@ -86,6 +85,7 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
replace_parameter(layer, "weight_scale", weight_scale.data)
|
||||
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -237,14 +237,12 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype)
|
||||
w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype)
|
||||
w13_weight, w2_weight = self.get_weights_for_quantization(layer)
|
||||
layer.w13_input_scale = None
|
||||
layer.w2_input_scale = None
|
||||
|
||||
w13, w13_scale = self._quantize_mxfp8_moe_weight(layer.w13_weight)
|
||||
w2, w2_scale = self._quantize_mxfp8_moe_weight(layer.w2_weight)
|
||||
w13, w13_scale = self._quantize_mxfp8_moe_weight(w13_weight)
|
||||
w2, w2_scale = self._quantize_mxfp8_moe_weight(w2_weight)
|
||||
|
||||
self._setup_kernel(
|
||||
layer,
|
||||
@@ -255,5 +253,6 @@ class Mxfp8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
layer.w13_input_scale,
|
||||
layer.w2_input_scale,
|
||||
)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
Reference in New Issue
Block a user