[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
@@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Reconstruct serialized weights for online requantization."""
|
||||
if self.wkey.key is kMxfp8Static:
|
||||
return dequant_mxfp8_to_bf16(
|
||||
layer.weight.contiguous(), layer.weight_scale.contiguous()
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"ModelOpt weight dequantization is only supported for MXFP8."
|
||||
)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
return self.fmt.apply(
|
||||
layer,
|
||||
|
||||
Reference in New Issue
Block a user