[Bugfix][Quantization] Register Quark per-block FP8 scales as weight_scale (#54770)
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
fxmarty-amd
Shanshan Shen
parent
6cbb3c154e
commit
385ba6b5b0
@@ -11,6 +11,7 @@ import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
@@ -702,6 +703,68 @@ def enable_pickle(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_registers_weight_scale(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes."
|
||||
"quark_w8a8_fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
|
||||
)
|
||||
scheme = QuarkW8A8Fp8PerBlock(kFp8Static128BlockSym, kFp8Dynamic128Sym)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_scale = torch.tensor([2.0])
|
||||
assert get_fp8_block_weight_scale(layer) is None
|
||||
layer.scheme = scheme
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
|
||||
layer.weight_scale_inv = torch.tensor([3.0])
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
|
||||
layer.scheme = None
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale_inv
|
||||
|
||||
loaded = torch.nn.Module()
|
||||
|
||||
def weight_loader(param, loaded_weight):
|
||||
return None
|
||||
|
||||
dummy_param = torch.nn.Parameter(torch.empty(1), requires_grad=False)
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"validate_fp8_block_shape"
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"create_fp8_weight_parameter",
|
||||
return_value=dummy_param,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"create_fp8_scale_parameter",
|
||||
return_value=dummy_param,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"init_fp8_linear_kernel",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
loaded,
|
||||
output_partition_sizes=[256],
|
||||
input_size_per_partition=256,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=weight_loader,
|
||||
input_size=256,
|
||||
output_size=256,
|
||||
)
|
||||
assert hasattr(loaded, "weight_scale")
|
||||
assert not hasattr(loaded, "weight_scale_inv")
|
||||
|
||||
|
||||
def test_quark_config_has_no_model_specific_fused_mappings():
|
||||
config = QuarkConfig({})
|
||||
|
||||
|
||||
@@ -230,6 +230,9 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
return QuarkW8A8Fp8.get_min_capability()
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
# Quark exports the dequant multiplier as ``weight_scale`` (the same
|
||||
# numerical convention as DeepSeek's ``weight_scale_inv``). Kernels
|
||||
# multiply ``weight * scale``; do not invert here.
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
@@ -280,8 +283,10 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
|
||||
weight_loader,
|
||||
scale_dtype=scale_dtype,
|
||||
)
|
||||
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
|
||||
layer.register_parameter("weight_scale_inv", weight_scale)
|
||||
# Match Quark-exported checkpoints (``.weight_scale``).
|
||||
# ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the
|
||||
# same semantic meaning as Quark's `weight_scale`.
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
assert self.activation_quant_key is not None
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
|
||||
@@ -50,6 +50,21 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool:
|
||||
return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
|
||||
|
||||
|
||||
def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None:
|
||||
"""Return the block-FP8 weight scale for supported quant methods."""
|
||||
# Local import avoids a circular import: quark_w8a8_fp8 imports this module.
|
||||
from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import (
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
getattr(layer, "scheme", None),
|
||||
QuarkW8A8Fp8PerBlock,
|
||||
):
|
||||
return getattr(layer, "weight_scale", None)
|
||||
return getattr(layer, "weight_scale_inv", None)
|
||||
|
||||
|
||||
def input_to_float8(
|
||||
x: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
@@ -123,20 +123,20 @@ class DeepseekV4MLP(nn.Module):
|
||||
# B-preshuffle the gate_up_proj weight in place (single weight).
|
||||
if not self._gateup:
|
||||
return
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w = getattr(self.gate_up_proj, "weight", None)
|
||||
ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale
|
||||
ws = get_fp8_block_weight_scale(self.gate_up_proj)
|
||||
if w is None or ws is None or w.dim() != 2:
|
||||
return
|
||||
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
|
||||
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
|
||||
return
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
|
||||
ws = _upcast_e8m0_to_fp32(ws).contiguous()
|
||||
replace_parameter(
|
||||
self.gate_up_proj,
|
||||
@@ -774,6 +774,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_param_name(name: str) -> str:
|
||||
# Fp8LinearMethod registers block scales as ``weight_scale_inv``.
|
||||
# Quark checkpoints / QuarkW8A8Fp8PerBlock use ``weight_scale``.
|
||||
# Alias only when the registered param is the ``_inv`` name.
|
||||
inv_name = f"{name}_inv"
|
||||
if name not in params_dict and inv_name in params_dict:
|
||||
return inv_name
|
||||
@@ -922,20 +925,17 @@ def _make_deepseek_v4_weights_mapper(
|
||||
) -> WeightsMapper:
|
||||
if expert_dtype == "fp4":
|
||||
# MXFP4 experts use Mxfp4MoEMethod, which registers scales as
|
||||
# ``w{1,2,3}_weight_scale`` (no _inv suffix). FP8 linear and
|
||||
# (non-fused) shared experts use Fp8LinearMethod's block scales,
|
||||
# which register as ``weight_scale_inv``.
|
||||
# ``w{1,2,3}_weight_scale`` (no _inv suffix). Native DeepSeek FP8
|
||||
# linear / shared experts use Fp8LinearMethod's ``weight_scale_inv``.
|
||||
# Quark linear block-FP8 registers ``weight_scale`` and is left
|
||||
# as-is; load_weights aliases ``.weight_scale`` -> ``.weight_scale_inv``
|
||||
# only when the latter is the registered parameter.
|
||||
#
|
||||
# - DeepSeek native ``.scale``: expert scales -> ``.weight_scale``,
|
||||
# everything else -> ``.weight_scale_inv``.
|
||||
# - AMD-Quark ``.weight_scale``: linear/attn scales ->
|
||||
# ``.weight_scale_inv``. Expert and shared-expert
|
||||
# ``w{1,2,3}.weight_scale`` are left untouched (consumed as-is by
|
||||
# the MXFP4 expert loader, which produces ``w{13,2}_weight_scale``);
|
||||
scale_regex = {
|
||||
re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale",
|
||||
re.compile(r"\.scale$"): ".weight_scale_inv",
|
||||
re.compile(r"(?<!\.w[123])\.weight_scale$"): ".weight_scale_inv",
|
||||
}
|
||||
else:
|
||||
# FP8 experts use Fp8MoEMethod (block_quant=True), which registers
|
||||
|
||||
@@ -335,11 +335,10 @@ class DeepSeekV4MTP(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
def _resolve_scale_name(name: str) -> str:
|
||||
# Quark checkpoints name FP8 block scales ``.weight_scale``,
|
||||
# but block-FP8 layers register them as ``.weight_scale_inv``
|
||||
# while MXFP4 experts register ``.weight_scale``. Auto-detect:
|
||||
# rename to ``_inv`` only when that variant exists and the plain
|
||||
# one does not.
|
||||
# Quark checkpoints and QuarkW8A8Fp8PerBlock use ``.weight_scale``.
|
||||
# Native block-FP8 layers register ``.weight_scale_inv``. Rename
|
||||
# to ``_inv`` only when that variant exists and the plain one
|
||||
# does not.
|
||||
if name.endswith(".weight_scale") and name not in params_dict:
|
||||
inv = name.removesuffix(".weight_scale") + ".weight_scale_inv"
|
||||
if inv in params_dict:
|
||||
|
||||
@@ -535,6 +535,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
return
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
@@ -545,7 +546,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
# K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold.
|
||||
if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0:
|
||||
return None
|
||||
ws = getattr(linear, "weight_scale_inv", None) # per-block scale
|
||||
ws = get_fp8_block_weight_scale(linear)
|
||||
if ws is None:
|
||||
return None
|
||||
if ws.dtype == torch.float8_e8m0fnu:
|
||||
|
||||
@@ -1202,14 +1202,17 @@ def _get_cached_wo_a_bf16(
|
||||
cached = getattr(wo_a, "_dsv4_wo_a_bf16", None)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if hasattr(wo_a, "weight_scale_inv"):
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
|
||||
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
|
||||
if wo_a_scale_param is not None:
|
||||
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
|
||||
torch.float32
|
||||
)
|
||||
wo_a_scale = _expand_2d_block_scales(
|
||||
wo_a.weight_scale_inv.view(
|
||||
n_local_groups, -1, wo_a.weight_scale_inv.shape[-1]
|
||||
),
|
||||
wo_a_scale_param.view(n_local_groups, -1, wo_a_scale_param.shape[-1]),
|
||||
o_lora_rank,
|
||||
hidden_dim,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user