[Bugfix][Quantization] Register Quark per-block FP8 scales as weight_scale (#54770)

Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
Co-authored-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
jimmy-adams
2026-09-05 10:44:24 +08:00
committed by GitHub
co-authored by fxmarty-amd Shanshan Shen
parent 6cbb3c154e
commit 385ba6b5b0
7 changed files with 111 additions and 25 deletions
@@ -230,6 +230,9 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
# Quark exports the dequant multiplier as ``weight_scale`` (the same
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
# multiply ``weight * scale``; do not invert here.
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
@@ -280,8 +283,10 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
weight_loader,
scale_dtype=scale_dtype,
)
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
# Match Quark-exported checkpoints (``.weight_scale``).
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
# same semantic meaning as Quark's `weight_scale`.
layer.register_parameter("weight_scale", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
@@ -50,6 +50,21 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
"""Return the block-FP8 weight scale for supported quant methods."""
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
QuarkW8A8Fp8PerBlock,
)
if isinstance(
getattr(layer, "scheme", None),
QuarkW8A8Fp8PerBlock,
):
return getattr(layer, "weight_scale", None)
return getattr(layer, "weight_scale_inv", None)
def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]: