diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 49af26fd88..848e095848 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -11,6 +11,7 @@ import importlib.metadata from dataclasses import dataclass from importlib.util import find_spec from types import SimpleNamespace +from unittest.mock import MagicMock, patch import huggingface_hub import lm_eval @@ -702,6 +703,68 @@ def enable_pickle(monkeypatch): monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") +def test_quark_w8a8_fp8_per_block_registers_weight_scale(monkeypatch): + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + get_fp8_block_weight_scale, + ) + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.quark.schemes." + "quark_w8a8_fp8.get_current_vllm_config", + lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)), + ) + scheme = QuarkW8A8Fp8PerBlock(kFp8Static128BlockSym, kFp8Dynamic128Sym) + + layer = torch.nn.Module() + layer.weight_scale = torch.tensor([2.0]) + assert get_fp8_block_weight_scale(layer) is None + layer.scheme = scheme + assert get_fp8_block_weight_scale(layer) is layer.weight_scale + layer.weight_scale_inv = torch.tensor([3.0]) + assert get_fp8_block_weight_scale(layer) is layer.weight_scale + layer.scheme = None + assert get_fp8_block_weight_scale(layer) is layer.weight_scale_inv + + loaded = torch.nn.Module() + + def weight_loader(param, loaded_weight): + return None + + dummy_param = torch.nn.Parameter(torch.empty(1), requires_grad=False) + with ( + patch( + "vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8." + "validate_fp8_block_shape" + ), + patch( + "vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8." + "create_fp8_weight_parameter", + return_value=dummy_param, + ), + patch( + "vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8." + "create_fp8_scale_parameter", + return_value=dummy_param, + ), + patch( + "vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8." + "init_fp8_linear_kernel", + return_value=MagicMock(), + ), + ): + scheme.create_weights( + loaded, + output_partition_sizes=[256], + input_size_per_partition=256, + params_dtype=torch.bfloat16, + weight_loader=weight_loader, + input_size=256, + output_size=256, + ) + assert hasattr(loaded, "weight_scale") + assert not hasattr(loaded, "weight_scale_inv") + + def test_quark_config_has_no_model_specific_fused_mappings(): config = QuarkConfig({}) diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index 40bdac7ee2..57b0836128 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -230,6 +230,9 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme): return QuarkW8A8Fp8.get_min_capability() def process_weights_after_loading(self, layer) -> None: + # Quark exports the dequant multiplier as ``weight_scale`` (the same + # numerical convention as DeepSeek's ``weight_scale_inv``). Kernels + # multiply ``weight * scale``; do not invert here. self.fp8_linear.process_weights_after_loading(layer) def create_weights( @@ -280,8 +283,10 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme): weight_loader, scale_dtype=scale_dtype, ) - # DeepSeek V4 weight mappers route checkpoint ".scale" tensors here. - layer.register_parameter("weight_scale_inv", weight_scale) + # Match Quark-exported checkpoints (``.weight_scale``). + # ``Fp8LinearMethod.create_weights`` registers weight_scale_inv, which has the + # same semantic meaning as Quark's `weight_scale`. + layer.register_parameter("weight_scale", weight_scale) assert self.activation_quant_key is not None self.fp8_linear = init_fp8_linear_kernel( diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index 0d49e3510f..451ae3a49a 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -50,6 +50,21 @@ def is_fp8(x: torch.dtype | torch.Tensor) -> bool: return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz +def get_fp8_block_weight_scale(layer: torch.nn.Module) -> torch.Tensor | None: + """Return the block-FP8 weight scale for supported quant methods.""" + # Local import avoids a circular import: quark_w8a8_fp8 imports this module. + from vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8 import ( + QuarkW8A8Fp8PerBlock, + ) + + if isinstance( + getattr(layer, "scheme", None), + QuarkW8A8Fp8PerBlock, + ): + return getattr(layer, "weight_scale", None) + return getattr(layer, "weight_scale_inv", None) + + def input_to_float8( x: torch.Tensor, dtype: torch.dtype | None = None ) -> tuple[torch.Tensor, torch.Tensor]: diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index 7a67c8f420..177775e413 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -123,20 +123,20 @@ class DeepseekV4MLP(nn.Module): # B-preshuffle the gate_up_proj weight in place (single weight). if not self._gateup: return + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + _upcast_e8m0_to_fp32, + get_fp8_block_weight_scale, + ) from vllm.model_executor.utils import replace_parameter w = getattr(self.gate_up_proj, "weight", None) - ws = getattr(self.gate_up_proj, "weight_scale_inv", None) # per-block scale + ws = get_fp8_block_weight_scale(self.gate_up_proj) if w is None or ws is None or w.dim() != 2: return # K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold. if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0: return if ws.dtype == torch.float8_e8m0fnu: - from vllm.model_executor.layers.quantization.utils.fp8_utils import ( - _upcast_e8m0_to_fp32, - ) - ws = _upcast_e8m0_to_fp32(ws).contiguous() replace_parameter( self.gate_up_proj, @@ -774,6 +774,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): loaded_params: set[str] = set() def _resolve_param_name(name: str) -> str: + # Fp8LinearMethod registers block scales as ``weight_scale_inv``. + # Quark checkpoints / QuarkW8A8Fp8PerBlock use ``weight_scale``. + # Alias only when the registered param is the ``_inv`` name. inv_name = f"{name}_inv" if name not in params_dict and inv_name in params_dict: return inv_name @@ -922,20 +925,17 @@ def _make_deepseek_v4_weights_mapper( ) -> WeightsMapper: if expert_dtype == "fp4": # MXFP4 experts use Mxfp4MoEMethod, which registers scales as - # ``w{1,2,3}_weight_scale`` (no _inv suffix). FP8 linear and - # (non-fused) shared experts use Fp8LinearMethod's block scales, - # which register as ``weight_scale_inv``. + # ``w{1,2,3}_weight_scale`` (no _inv suffix). Native DeepSeek FP8 + # linear / shared experts use Fp8LinearMethod's ``weight_scale_inv``. + # Quark linear block-FP8 registers ``weight_scale`` and is left + # as-is; load_weights aliases ``.weight_scale`` -> ``.weight_scale_inv`` + # only when the latter is the registered parameter. # # - DeepSeek native ``.scale``: expert scales -> ``.weight_scale``, # everything else -> ``.weight_scale_inv``. - # - AMD-Quark ``.weight_scale``: linear/attn scales -> - # ``.weight_scale_inv``. Expert and shared-expert - # ``w{1,2,3}.weight_scale`` are left untouched (consumed as-is by - # the MXFP4 expert loader, which produces ``w{13,2}_weight_scale``); scale_regex = { re.compile(r"(\.experts\.\d+\.w[123])\.scale$"): r"\1.weight_scale", re.compile(r"\.scale$"): ".weight_scale_inv", - re.compile(r"(? str: - # Quark checkpoints name FP8 block scales ``.weight_scale``, - # but block-FP8 layers register them as ``.weight_scale_inv`` - # while MXFP4 experts register ``.weight_scale``. Auto-detect: - # rename to ``_inv`` only when that variant exists and the plain - # one does not. + # Quark checkpoints and QuarkW8A8Fp8PerBlock use ``.weight_scale``. + # Native block-FP8 layers register ``.weight_scale_inv``. Rename + # to ``_inv`` only when that variant exists and the plain one + # does not. if name.endswith(".weight_scale") and name not in params_dict: inv = name.removesuffix(".weight_scale") + ".weight_scale_inv" if inv in params_dict: diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 3c8d23c4b4..3b1df97dc7 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -535,6 +535,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): return from vllm.model_executor.layers.quantization.utils.fp8_utils import ( _upcast_e8m0_to_fp32, + get_fp8_block_weight_scale, ) from vllm.model_executor.utils import replace_parameter @@ -545,7 +546,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): # K % 128 (group-128 quant) and N % 16 (shuffle_weight) must hold. if w.shape[-1] % 128 != 0 or w.shape[0] % 16 != 0: return None - ws = getattr(linear, "weight_scale_inv", None) # per-block scale + ws = get_fp8_block_weight_scale(linear) if ws is None: return None if ws.dtype == torch.float8_e8m0fnu: diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 62f91f260e..33f6386e34 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -1202,14 +1202,17 @@ def _get_cached_wo_a_bf16( cached = getattr(wo_a, "_dsv4_wo_a_bf16", None) if cached is not None: return cached - if hasattr(wo_a, "weight_scale_inv"): + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + get_fp8_block_weight_scale, + ) + + wo_a_scale_param = get_fp8_block_weight_scale(wo_a) + if wo_a_scale_param is not None: wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to( torch.float32 ) wo_a_scale = _expand_2d_block_scales( - wo_a.weight_scale_inv.view( - n_local_groups, -1, wo_a.weight_scale_inv.shape[-1] - ), + wo_a_scale_param.view(n_local_groups, -1, wo_a_scale_param.shape[-1]), o_lora_rank, hidden_dim, )