Fused Shared Expert Support for AMD Quark DeepSeek-V4 Model Checkpoints (#48044)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Colin Z
2026-07-23 00:34:17 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 229e01e9e1
commit 27ffbfde8d
3 changed files with 175 additions and 13 deletions
+37 -2
View File
@@ -4,7 +4,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, cast
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.fused_moe import (
@@ -117,13 +117,29 @@ class DeepseekV4FP8Config(Fp8Config):
def get_name(cls) -> QuantizationMethods:
return "deepseek_v4_fp8"
@staticmethod
def _is_quark_mxfp4_ocp(hf_quant_cfg: dict) -> bool:
"""True for AMD-Quark exports whose global scheme is MXFP4."""
weight = (hf_quant_cfg.get("global_quant_config") or {}).get("weight") or {}
return (
weight.get("dtype") == "fp4"
and weight.get("qscheme") == "per_group"
and weight.get("group_size") == 32
)
@classmethod
def override_quantization_method(
cls, hf_quant_cfg, user_quant, hf_config=None
) -> QuantizationMethods | None:
if not (
isinstance(hf_quant_cfg, dict)
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
and (
hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
or (
hf_quant_cfg.get("quant_method") == "quark"
and cls._is_quark_mxfp4_ocp(hf_quant_cfg)
)
)
):
return None
model_type = getattr(hf_config, "model_type", None)
@@ -131,6 +147,25 @@ class DeepseekV4FP8Config(Fp8Config):
return "deepseek_v4_fp8"
return None
@classmethod
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
# so translate the schema into format Fp8Config.from_config expects.
if config.get("quant_method") == "quark":
quark_exclude = config.get("exclude") or []
config = {
"quant_method": "fp8",
"activation_scheme": "dynamic",
"fmt": "e4m3",
"scale_fmt": "ue8m0",
"weight_block_size": [128, 128],
"ignored_layers": [
name for name in quark_exclude if isinstance(name, str)
],
}
return cast("DeepseekV4FP8Config", super().from_config(config))
def get_quant_method(self, layer, prefix):
if isinstance(layer, RoutedExperts):
if is_layer_skipped(