Fused Shared Expert Support for AMD Quark DeepSeek-V4 Model Checkpoints (#48044)
Signed-off-by: Colin Zeng <Colin.Zeng@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
229e01e9e1
commit
27ffbfde8d
@@ -4,7 +4,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -117,13 +117,29 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
def get_name(cls) -> QuantizationMethods:
|
||||
return "deepseek_v4_fp8"
|
||||
|
||||
@staticmethod
|
||||
def _is_quark_mxfp4_ocp(hf_quant_cfg: dict) -> bool:
|
||||
"""True for AMD-Quark exports whose global scheme is MXFP4."""
|
||||
weight = (hf_quant_cfg.get("global_quant_config") or {}).get("weight") or {}
|
||||
return (
|
||||
weight.get("dtype") == "fp4"
|
||||
and weight.get("qscheme") == "per_group"
|
||||
and weight.get("group_size") == 32
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
if not (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
and (
|
||||
hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
or (
|
||||
hf_quant_cfg.get("quant_method") == "quark"
|
||||
and cls._is_quark_mxfp4_ocp(hf_quant_cfg)
|
||||
)
|
||||
)
|
||||
):
|
||||
return None
|
||||
model_type = getattr(hf_config, "model_type", None)
|
||||
@@ -131,6 +147,25 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
return "deepseek_v4_fp8"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict) -> DeepseekV4FP8Config:
|
||||
# Reroute AMD-Quark fused shared expert MXFP4 checkpoints onto the fp8
|
||||
# path: the runtime layout matches the DeepSeek-native fp8 checkpoint,
|
||||
# so translate the schema into format Fp8Config.from_config expects.
|
||||
if config.get("quant_method") == "quark":
|
||||
quark_exclude = config.get("exclude") or []
|
||||
config = {
|
||||
"quant_method": "fp8",
|
||||
"activation_scheme": "dynamic",
|
||||
"fmt": "e4m3",
|
||||
"scale_fmt": "ue8m0",
|
||||
"weight_block_size": [128, 128],
|
||||
"ignored_layers": [
|
||||
name for name in quark_exclude if isinstance(name, str)
|
||||
],
|
||||
}
|
||||
return cast("DeepseekV4FP8Config", super().from_config(config))
|
||||
|
||||
def get_quant_method(self, layer, prefix):
|
||||
if isinstance(layer, RoutedExperts):
|
||||
if is_layer_skipped(
|
||||
|
||||
Reference in New Issue
Block a user