[Bugfix][ROCm] AMD-Quark mixed-precision DeepSeek-V4.1 support (#57071)

Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiao-llm
2026-09-25 20:51:18 -05:00
committed by GitHub
co-authored by Cursor
parent b6d2b4fc25
commit 5840d95284
11 changed files with 645 additions and 100 deletions
@@ -44,7 +44,12 @@ def get_quark_ocp_mx_group_size(
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
group_size = weight_quant.get("group_size")
if group_size is None:
block_size = weight_quant.get("block_size")
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
group_size = block_size[1]
return int(group_size) if group_size is not None else None
def is_shared_expert_quant_fse_compatible(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -11,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
kMxfp8Dynamic,
kMxfp8Static,
)
logger = init_logger(__name__)
@@ -21,14 +24,19 @@ _WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
"mxfp8_e4m3": kMxfp8Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
"mxfp8_e4m3": kMxfp8Dynamic,
}
# Unpacked OCP MX dtypes: one byte per element, no `packed_factor`.
OCP_MX_UNPACKED_DTYPES = {"mxfp8_e4m3"}
OCP_MX_DTYPES = {
"mxfp4",
"mxfp6_e3m2",
@@ -39,6 +47,56 @@ OCP_MX_DTYPES = {
}
def ocp_mx_weight_dtype_and_rows(
weight_quant: dict[str, Any] | None,
) -> tuple[str, int] | None:
"""Normalize a Quark weight quant config to ``(mx_dtype, scale_block_rows)``.
Quark spells OCP MX weights two ways. The canonical one is 1-D per-group:
``qscheme="per_group"``, ``group_size=32``, ``scale_format="e8m0"``. MXFP8
checkpoints may instead use a 2-D per-block spelling
(``qscheme="per_block"``, ``block_size=[R, 32]``,
``scale_type="float8_e8m0fnu"``), which carries one scale per ``R`` weight
rows rather than one per row. ``scale_block_rows`` is that ``R``; it is 1
for the canonical spelling, where the two layouts coincide.
Returns ``None`` when the config is not an OCP MX weight quantization.
"""
if not isinstance(weight_quant, dict):
return None
dtype = weight_quant.get("dtype")
if not isinstance(dtype, str):
return None
mx_dtype = dtype.replace("fp", "mxfp")
if mx_dtype not in _WEIGHT_QUANT_KEY_MAP:
return None
qscheme = weight_quant.get("qscheme")
if qscheme == "per_group":
if weight_quant.get("group_size") != OCP_MX_BLOCK_SIZE:
return None
if weight_quant.get("scale_format") != "e8m0":
return None
return mx_dtype, 1
if qscheme == "per_block":
# Only the unpacked dtypes are accepted here: expanding a 2-D scale
# over a sub-byte packed weight has no checkpoint to validate against.
if mx_dtype not in OCP_MX_UNPACKED_DTYPES:
return None
block_size = list(weight_quant.get("block_size") or [])
if len(block_size) != 2 or block_size[1] != OCP_MX_BLOCK_SIZE:
return None
if weight_quant.get("scale_type") != "float8_e8m0fnu":
return None
if weight_quant.get("symmetric") is not True or weight_quant.get("is_dynamic"):
return None
return mx_dtype, block_size[0]
return None
class OCP_MX_Scheme(str, Enum):
w_mxfp4 = "w_mxfp4"
w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4"