[Bugfix][ROCm] AMD-Quark mixed-precision DeepSeek-V4.1 support (#57071)

Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
xiao-llm
2026-09-25 20:51:18 -05:00
committed by GitHub
co-authored by Cursor
parent b6d2b4fc25
commit 5840d95284
11 changed files with 645 additions and 100 deletions
@@ -45,6 +45,8 @@ from vllm.model_executor.layers.quantization.quark.utils import (
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
ocp_mx_weight_dtype_and_rows,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -158,6 +160,20 @@ class QuarkConfig(QuantizationConfig):
self.quant_config = quant_config_with_hf_to_vllm_mapper
def _scale_block_rows(self, prefix: str, layer_type: type[torch.nn.Module]) -> int:
"""How many weight rows one checkpoint scale row covers.
1 for the canonical 1-D per-group MX spelling; ``block_size[0]`` for
MXFP8 exports that spell the scale 2-D (DeepSeek-V4.1 uses 32). The
scheme cannot work this out itself: it has no handle on the layer name,
and tensor-parallel sharding narrows the loaded scale so the ratio is
not recoverable from shapes either.
"""
config = self._find_matched_config(prefix, layer_type)
weight = self._unwrap_single_quant_config(config.get("weight"))
normalized = ocp_mx_weight_dtype_and_rows(weight)
return normalized[1] if normalized is not None else 1
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
@@ -175,6 +191,11 @@ class QuarkConfig(QuantizationConfig):
scheme_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
scale_block_rows=(
self._scale_block_rows(prefix, type(layer))
if scheme_cls is QuarkOCP_MX
else 1
),
weight_config=self._find_matched_config(prefix, type(layer)).get(
"weight"
),
@@ -640,52 +661,18 @@ class QuarkConfig(QuantizationConfig):
The rationale for checking only the weight type is that
the model loading concept and process primarily concerns the weights themselves.
"""
# Confirm weights quantized.
if not isinstance(weight_quant, dict):
# Confirm weights quantized, and normalize the two OCP MX config
# dialects (1-D per_group, and MXFP8's 2-D per_block) to a dtype.
normalized = ocp_mx_weight_dtype_and_rows(weight_quant)
if normalized is None:
logger.debug(
"Quark model's weight quantization is incompatible with OCP_MX format: "
"weight_quant is not a dictionary."
"Quark model's weight quantization is incompatible with OCP MX "
"format: %s",
weight_quant,
)
return QuantKeyMatch(False, None, None)
# Input and weight qscheme needs to be per group.
if weight_quant.get("qscheme") != "per_group":
logger.debug(
"Quark model's weight quantization is incompatible with OCP MX format: "
"weight is not per_group."
)
return QuantKeyMatch(False, None, None)
# Input and weight group size needs to be 32.
if weight_quant.get("group_size") != 32:
logger.debug(
"Quark model's weight quantization is incompatible with OCP MX format: "
"group_size of weight is not 32."
)
return QuantKeyMatch(False, None, None)
# Activations and weight scales need to be in e8m0 format.
if weight_quant.get("scale_format") != "e8m0":
logger.debug(
"Quark model's weight quantization is incompatible with OCP MX format: "
"scale_format of weight is not e8m0."
)
return QuantKeyMatch(False, None, None)
# Input and weight dtypes need to be any of fp4,
# fp6_e3m2 or fp6_e3m2, possibly mixed.
if weight_quant.get("dtype") not in {
"fp4",
"fp6_e3m2",
"fp6_e2m3",
}:
logger.debug(
"Quark model's weight quantization is incompatible with OCP MX format: "
"dtype is not in {fp4, fp6_e3m2, fp6_e2m3}."
)
return QuantKeyMatch(False, None, None)
weight_dtype = weight_quant["dtype"].replace("fp", "mxfp")
weight_dtype, _ = normalized
weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype]
if input_quant is None:
activation_quant_key = None
@@ -703,6 +690,14 @@ class QuarkConfig(QuantizationConfig):
"input scales."
)
return QuantKeyMatch(False, None, None)
elif (
input_quant["dtype"] == "fp8_e4m3"
and input_quant.get("qscheme") == "per_group"
and input_quant.get("group_size") == OCP_MX_BLOCK_SIZE
):
# MXFP8 activations. Must be checked before the per-tensor FP8
# branch below, which would otherwise claim them.
activation_quant_key = _ACTIVATION_QUANT_KEY_MAP["mxfp8_e4m3"]
elif input_quant["dtype"] == "fp8_e4m3":
activation_quant_key = kFp8DynamicTensorSym
else:
@@ -903,6 +898,7 @@ class QuarkConfig(QuantizationConfig):
scheme_cls: type["QuarkScheme"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
scale_block_rows: int = 1,
weight_config: dict[str, Any] | None = None,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
@@ -942,6 +938,9 @@ class QuarkConfig(QuantizationConfig):
):
kwargs["weight_quant_key"] = weight_quant_key
if scheme_cls is QuarkOCP_MX:
kwargs["scale_block_rows"] = scale_block_rows
scheme = scheme_cls(**kwargs)
# Raise error if device does not support the scheme
@@ -1587,7 +1587,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
raise ValueError(
f"Unsupported OCP MX dtype combination for MoE: "
f"input_dtype={self.input_dtype}, weight_dtype={self.weight_dtype}. "
f"Please check that the combination is supported in OCP_MX_Scheme."
f"MXFP8 experts are not supported through the MoE path; only "
f"the linear path handles MXFP8. Supported MoE weight dtypes "
f"are mxfp4, mxfp6_e3m2, and mxfp6_e2m3."
)
# TODO(bowenbao): refactor and introduce backends for other OCP MX schemes,
@@ -1676,10 +1678,16 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
if quant_dtype == "mxfp4":
assert dim % 2 == 0
return dim // 2
else:
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
# FP6 packs 4 * 6 = 24 bits on 3 bytes.
assert (dim * 3) % 4 == 0
return (dim * 3) // 4
else:
raise NotImplementedError(
f"Unsupported quant_dtype in QuarkOCP_MX_MoEMethod."
f"get_packed_dim: {quant_dtype}. MXFP8 is not supported for "
f"MoE experts."
)
def create_weights(
self,
@@ -10,22 +10,31 @@ from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
MxFp4LinearKernel,
MxFp6LinearKernel,
Mxfp8LinearKernel,
init_mxfp4_linear_kernel,
init_mxfp6_linear_kernel,
init_mxfp8_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
OCP_MX_UNPACKED_DTYPES,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kMxfp6E2M3Static,
kMxfp6E3M2Static,
kMxfp8Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
PackedvLLMParameter,
)
from vllm.platforms import current_platform
@@ -36,7 +45,7 @@ logger = init_logger(__name__)
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
@@ -44,9 +53,10 @@ class QuarkOCP_MX(QuarkScheme):
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
scale_block_rows: int = 1,
):
super().__init__(weight_quant_key, activation_quant_key)
self.scale_block_rows = scale_block_rows
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
@@ -62,11 +72,21 @@ class QuarkOCP_MX(QuarkScheme):
else None
)
if self.weight_dtype == "mxfp4":
self.packed_factor: int | Fraction = 2
self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES
if self.is_unpacked:
self.packed_factor: int | Fraction = 1
elif self.weight_dtype == "mxfp4":
self.packed_factor = 2
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
if self.is_unpacked:
# MXFP8 runs on the shared kernel abstraction, which picks a native
# backend where one exists and emulates otherwise. The MXFP4/MXFP6
# warnings below do not apply.
return
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
@@ -88,7 +108,9 @@ class QuarkOCP_MX(QuarkScheme):
)
def get_packed_dim(self, dim: int, quant_dtype: str):
if quant_dtype == "mxfp4":
if quant_dtype in OCP_MX_UNPACKED_DTYPES:
return dim
elif quant_dtype == "mxfp4":
assert dim % 2 == 0
return dim // 2
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
@@ -102,6 +124,26 @@ class QuarkOCP_MX(QuarkScheme):
"open an issue."
)
def _expanding_scale_loader(self, weight_loader: Callable) -> Callable:
"""Expand a 2-D block scale to one row per weight row.
A ``block_size=[R, 32]`` export carries one e8m0 value per R weight
rows; the kernels want one per row. At ``R == 1`` the layouts already
coincide, so the loader is passed through untouched.
"""
if self.scale_block_rows == 1:
return weight_loader
rows = self.scale_block_rows
def expanding_loader(param, loaded_weight, *args, **kwargs):
loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave(
rows, dim=0
)
return weight_loader(param, loaded_weight, *args, **kwargs)
return expanding_loader
@classmethod
def get_min_capability(cls) -> int:
return 70
@@ -130,36 +172,68 @@ class QuarkOCP_MX(QuarkScheme):
"tensor-parallel sharding for this layer."
)
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
if self.is_unpacked:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
layer.input_size_per_partition = input_size_per_partition
layer.output_size_per_partition = output_size_per_partition
layer.params_dtype = params_dtype
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
weight = ModelWeightParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition,
dtype=MXFP8_VALUE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=MXFP8_SCALE_DTYPE,
),
input_dim=1,
output_dim=0,
weight_loader=self._expanding_scale_loader(weight_loader),
)
layer.register_parameter("weight_scale", weight_scale)
layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE]
else:
output_size_per_partition = sum(output_partition_sizes)
layer.logical_widths = output_partition_sizes
# WEIGHT
weight = PackedvLLMParameter(
data=torch.empty(
output_size_per_partition,
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
packed_dim=1,
packed_factor=self.packed_factor,
weight_loader=weight_loader,
)
layer.register_parameter("weight", weight)
# WEIGHT SCALE
weight_scale = GroupQuantScaleParameter(
data=torch.empty(
output_size_per_partition,
input_size_per_partition // OCP_MX_BLOCK_SIZE,
dtype=torch.uint8,
),
input_dim=1,
output_dim=0,
weight_loader=weight_loader,
)
layer.register_parameter("weight_scale", weight_scale)
if self.weight_quant_key == kMxfp4Static:
self.ocp_mx_linear = init_mxfp4_linear_kernel(
@@ -170,6 +244,16 @@ class QuarkOCP_MX(QuarkScheme):
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.activation_quant_key,
)
elif self.weight_quant_key == kMxfp8Static:
self.ocp_mx_linear = init_mxfp8_linear_kernel(
weight_shape=layer.weight.shape
)
else:
raise NotImplementedError(
"No OCP MX linear kernel for weight_quant_key="
f"{self.weight_quant_key}. Something is wrong, please open an "
"issue."
)
def apply_weights(
self,
@@ -44,7 +44,12 @@ def get_quark_ocp_mx_group_size(
return None
assert weight_quant is not None
return int(weight_quant["group_size"])
group_size = weight_quant.get("group_size")
if group_size is None:
block_size = weight_quant.get("block_size")
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
group_size = block_size[1]
return int(group_size) if group_size is not None else None
def is_shared_expert_quant_fse_compatible(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum
from typing import Any
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -11,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
kMxfp8Dynamic,
kMxfp8Static,
)
logger = init_logger(__name__)
@@ -21,14 +24,19 @@ _WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
"mxfp8_e4m3": kMxfp8Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
"mxfp8_e4m3": kMxfp8Dynamic,
}
# Unpacked OCP MX dtypes: one byte per element, no `packed_factor`.
OCP_MX_UNPACKED_DTYPES = {"mxfp8_e4m3"}
OCP_MX_DTYPES = {
"mxfp4",
"mxfp6_e3m2",
@@ -39,6 +47,56 @@ OCP_MX_DTYPES = {
}
def ocp_mx_weight_dtype_and_rows(
weight_quant: dict[str, Any] | None,
) -> tuple[str, int] | None:
"""Normalize a Quark weight quant config to ``(mx_dtype, scale_block_rows)``.
Quark spells OCP MX weights two ways. The canonical one is 1-D per-group:
``qscheme="per_group"``, ``group_size=32``, ``scale_format="e8m0"``. MXFP8
checkpoints may instead use a 2-D per-block spelling
(``qscheme="per_block"``, ``block_size=[R, 32]``,
``scale_type="float8_e8m0fnu"``), which carries one scale per ``R`` weight
rows rather than one per row. ``scale_block_rows`` is that ``R``; it is 1
for the canonical spelling, where the two layouts coincide.
Returns ``None`` when the config is not an OCP MX weight quantization.
"""
if not isinstance(weight_quant, dict):
return None
dtype = weight_quant.get("dtype")
if not isinstance(dtype, str):
return None
mx_dtype = dtype.replace("fp", "mxfp")
if mx_dtype not in _WEIGHT_QUANT_KEY_MAP:
return None
qscheme = weight_quant.get("qscheme")
if qscheme == "per_group":
if weight_quant.get("group_size") != OCP_MX_BLOCK_SIZE:
return None
if weight_quant.get("scale_format") != "e8m0":
return None
return mx_dtype, 1
if qscheme == "per_block":
# Only the unpacked dtypes are accepted here: expanding a 2-D scale
# over a sub-byte packed weight has no checkpoint to validate against.
if mx_dtype not in OCP_MX_UNPACKED_DTYPES:
return None
block_size = list(weight_quant.get("block_size") or [])
if len(block_size) != 2 or block_size[1] != OCP_MX_BLOCK_SIZE:
return None
if weight_quant.get("scale_type") != "float8_e8m0fnu":
return None
if weight_quant.get("symmetric") is not True or weight_quant.get("is_dynamic"):
return None
return mx_dtype, block_size[0]
return None
class OCP_MX_Scheme(str, Enum):
w_mxfp4 = "w_mxfp4"
w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4"