[Bugfix][Quantization] Register Quark per-block FP8 scales as weight_scale (#54770)

Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com>
Co-authored-by: fxmarty-amd <felmarty@amd.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
jimmy-adams
2026-09-05 10:44:24 +08:00
committed by GitHub
co-authored by fxmarty-amd Shanshan Shen
parent 6cbb3c154e
commit 385ba6b5b0
7 changed files with 111 additions and 25 deletions
+63
View File
@@ -11,6 +11,7 @@ import importlib.metadata
from dataclasses import dataclass
from importlib.util import find_spec
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import huggingface_hub
import lm_eval
@@ -702,6 +703,68 @@ def enable_pickle(monkeypatch):
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def test_quark_w8a8_fp8_per_block_registers_weight_scale(monkeypatch):
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.quark.schemes."
"quark_w8a8_fp8.get_current_vllm_config",
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
)
scheme = QuarkW8A8Fp8PerBlock(kFp8Static128BlockSym, kFp8Dynamic128Sym)
layer = torch.nn.Module()
layer.weight_scale = torch.tensor([2.0])
assert get_fp8_block_weight_scale(layer) is None
layer.scheme = scheme
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
layer.weight_scale_inv = torch.tensor([3.0])
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
layer.scheme = None
assert get_fp8_block_weight_scale(layer) is layer.weight_scale_inv
loaded = torch.nn.Module()
def weight_loader(param, loaded_weight):
return None
dummy_param = torch.nn.Parameter(torch.empty(1), requires_grad=False)
with (
patch(
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
"validate_fp8_block_shape"
),
patch(
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
"create_fp8_weight_parameter",
return_value=dummy_param,
),
patch(
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
"create_fp8_scale_parameter",
return_value=dummy_param,
),
patch(
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
"init_fp8_linear_kernel",
return_value=MagicMock(),
),
):
scheme.create_weights(
loaded,
output_partition_sizes=[256],
input_size_per_partition=256,
params_dtype=torch.bfloat16,
weight_loader=weight_loader,
input_size=256,
output_size=256,
)
assert hasattr(loaded, "weight_scale")
assert not hasattr(loaded, "weight_scale_inv")
def test_quark_config_has_no_model_specific_fused_mappings():
config = QuarkConfig({})
@@ -230,6 +230,9 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
return QuarkW8A8Fp8.get_min_capability()
def process_weights_after_loading(self, layer) -> None:
# Quark exports the dequant multiplier as ``weight_scale`` (the same
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
# multiply ``weight * scale``; do not invert here.
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
@@ -280,8 +283,10 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
weight_loader,
scale_dtype=scale_dtype,
)
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
# Match Quark-exported checkpoints (``.weight_scale``).
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
# same semantic meaning as Quark's `weight_scale`.
layer.register_parameter("weight_scale", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
@@ -50,6 +50,21 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
"""Return the block-FP8 weight scale for supported quant methods."""
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
QuarkW8A8Fp8PerBlock,
)
if isinstance(
getattr(layer, "scheme", None),
QuarkW8A8Fp8PerBlock,
):
return getattr(layer, "weight_scale", None)
return getattr(layer, "weight_scale_inv", None)
def input_to_float8(
x: torch.Tensor, dtype: torch.dtype | None = None
) -> tuple[torch.Tensor, torch.Tensor]:
+13 -13
View File
@@ -123,20 +123,20 @@ class DeepseekV4MLP(nn.Module):
# B-preshuffle the gate_up_proj weight in place (single weight).
if not self._gateup:
return
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
get_fp8_block_weight_scale,
)
from vllm.model_executor.utils import replace_parameter
w = getattr(self.gate_up_proj, "weight", None)
ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale
ws = get_fp8_block_weight_scale(self.gate_up_proj)
if w is None or ws is None or w.dim() != 2:
return
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
return
if ws.dtype == torch.float8_e8m0fnu:
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
ws = _upcast_e8m0_to_fp32(ws).contiguous()
replace_parameter(
self.gate_up_proj,
@@ -774,6 +774,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
loaded_params: set[str] = set()
def _resolve_param_name(name: str) -> str:
# Fp8LinearMethod registers block scales as ``weight_scale_inv``.
# Quark checkpoints / QuarkW8A8Fp8PerBlock use ``weight_scale``.
# Alias only when the registered param is the ``_inv`` name.
inv_name = f"{name}_inv"
if name not in params_dict and inv_name in params_dict:
return inv_name
@@ -922,20 +925,17 @@ def _make_deepseek_v4_weights_mapper(
) -> WeightsMapper:
if expert_dtype == "fp4":
# MXFP4 experts use Mxfp4MoEMethod, which registers scales as
# ``w{1,2,3}_weight_scale`` (no _inv suffix). FP8 linear and
# (non-fused) shared experts use Fp8LinearMethod's block scales,
# which register as ``weight_scale_inv``.
# ``w{1,2,3}_weight_scale`` (no _inv suffix). Native DeepSeek FP8
# linear / shared experts use Fp8LinearMethod's ``weight_scale_inv``.
# Quark linear block-FP8 registers ``weight_scale`` and is left
# as-is; load_weights aliases ``.weight_scale`` -> ``.weight_scale_inv``
# only when the latter is the registered parameter.
#
# - DeepSeek native ``.scale``: expert scales -> ``.weight_scale``,
# everything else -> ``.weight_scale_inv``.
# - AMD-Quark ``.weight_scale``: linear/attn scales ->
# ``.weight_scale_inv``. Expert and shared-expert
# ``w{1,2,3}.weight_scale`` are left untouched (consumed as-is by
# the MXFP4 expert loader, which produces ``w{13,2}_weight_scale``);
scale_regex = {
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
re.compile(r"\.scale$"): ".weight_scale_inv",
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
}
else:
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
+4 -5
View File
@@ -335,11 +335,10 @@ class DeepSeekV4MTP(nn.Module):
loaded_params: set[str] = set()
def _resolve_scale_name(name: str) -> str:
# Quark checkpoints name FP8 block scales ``.weight_scale``,
# but block-FP8 layers register them as ``.weight_scale_inv``
# while MXFP4 experts register ``.weight_scale``. Auto-detect:
# rename to ``_inv`` only when that variant exists and the plain
# one does not.
# Quark checkpoints and QuarkW8A8Fp8PerBlock use ``.weight_scale``.
# Native block-FP8 layers register ``.weight_scale_inv``. Rename
# to ``_inv`` only when that variant exists and the plain one
# does not.
if name.endswith(".weight_scale") and name not in params_dict:
inv = name.removesuffix(".weight_scale") + ".weight_scale_inv"
if inv in params_dict:
+2 -1
View File
@@ -535,6 +535,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
return
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
get_fp8_block_weight_scale,
)
from vllm.model_executor.utils import replace_parameter
@@ -545,7 +546,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
return None
ws = getattr(linear, "weight_scale_inv", None) # per-block scale
ws = get_fp8_block_weight_scale(linear)
if ws is None:
return None
if ws.dtype == torch.float8_e8m0fnu:
@@ -1202,14 +1202,17 @@ def _get_cached_wo_a_bf16(
cached = getattr(wo_a, "_dsv4_wo_a_bf16", None)
if cached is not None:
return cached
if hasattr(wo_a, "weight_scale_inv"):
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
if wo_a_scale_param is not None:
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
torch.float32
)
wo_a_scale = _expand_2d_block_scales(
wo_a.weight_scale_inv.view(
n_local_groups, -1, wo_a.weight_scale_inv.shape[-1]
),
wo_a_scale_param.view(n_local_groups, -1, wo_a_scale_param.shape[-1]),
o_lora_rank,
hidden_dim,
)