[Bugfix][Quantization] Register Quark per-block FP8 scales as weight_scale (#54770)
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
fxmarty-amd
Shanshan Shen
parent
6cbb3c154e
commit
385ba6b5b0
@@ -230,6 +230,9 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
return QuarkW8A8Fp8.get_min_capability()
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
# Quark exports the dequant multiplier as ``weight_scale`` (the same
|
||||
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
|
||||
# multiply ``weight * scale``; do not invert here.
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
@@ -280,8 +283,10 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
weight_loader,
|
||||
scale_dtype=scale_dtype,
|
||||
)
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
# Match Quark-exported checkpoints (``.weight_scale``).
|
||||
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
|
||||
# same semantic meaning as Quark's `weight_scale`.
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
|
||||
@@ -50,6 +50,21 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
|
||||
return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
|
||||
|
||||
|
||||
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
|
||||
"""Return the block-FP8 weight scale for supported quant methods."""
|
||||
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
getattr(layer, "scheme", None),
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
):
|
||||
return getattr(layer, "weight_scale", None)
|
||||
return getattr(layer, "weight_scale_inv", None)
|
||||
|
||||
|
||||
def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user