[Bugfix][ROCm] AMD-Quark mixed-precision DeepSeek-V4.1 support (#57071)
Signed-off-by: Xiao Yu <xiao.yu.dc@outlook.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -228,6 +228,85 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales(
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_vl_exposes_quant_mappings_on_class():
|
||||
"""``configure_quant_config`` reads both mappings off the class, before
|
||||
``__init__`` builds the instance mapper. Without them a Quark config's
|
||||
per-layer keys never match this wrapper's ``language_model.``-rooted
|
||||
prefixes and every attention shard falls back to the global spec."""
|
||||
from vllm.models.deepseek_v41.amd.vl_model import DeepseekV41ForCausalLM
|
||||
|
||||
assert "fused_wqa_wkv" in DeepseekV41ForCausalLM.packed_modules_mapping
|
||||
mapper = DeepseekV41ForCausalLM.hf_to_vllm_mapper.get_rename_mapper()
|
||||
assert mapper.apply_list(["layers.0.attn.wq_a"]) == [
|
||||
"language_model.model.layers.0.attn.wq_a"
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_engram_scale_accepts_quark_name():
|
||||
"""Quark exports name the engram scale ``embed.weight_scale``; the
|
||||
``\\.scale$`` rules only match a literal ``.scale`` suffix, so without an
|
||||
explicit rule the tensor is never routed and loading fails."""
|
||||
from vllm.models.deepseek_v41.amd import model as model_module
|
||||
|
||||
mapper = model_module._make_deepseek_v4_weights_mapper("fp4", "weight_scale")
|
||||
weight = torch.empty(0)
|
||||
mapped = [
|
||||
name
|
||||
for name, _ in mapper.apply(
|
||||
[
|
||||
("layers.1.engram.embed.weight_scale", weight),
|
||||
("layers.1.engram.embed.scale", weight),
|
||||
]
|
||||
)
|
||||
]
|
||||
assert mapped == [
|
||||
"model.layers.1.engram.embed_tokens.weight_scale_inv",
|
||||
"model.layers.1.engram.embed_tokens.weight_scale_inv",
|
||||
]
|
||||
|
||||
|
||||
def test_deepseek_v41_declines_quark_configs():
|
||||
"""``from_config`` rewrites a Quark config into a single global FP8 scheme,
|
||||
which would discard the per-layer specs of a mixed-precision export (MXFP4
|
||||
experts + 2-D block MXFP8 attention in DeepSeek-V4.1-Flash). QuarkConfig
|
||||
knows how to dispatch each scheme, so it must handle every Quark export."""
|
||||
from vllm.models.deepseek_v41.quant_config import DeepseekV4FP8Config
|
||||
|
||||
hf_config = SimpleNamespace(model_type="deepseek_v41")
|
||||
mxfp4_global = {
|
||||
"global_quant_config": {
|
||||
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32}
|
||||
},
|
||||
"quant_method": "quark",
|
||||
}
|
||||
|
||||
assert (
|
||||
DeepseekV4FP8Config.override_quantization_method(
|
||||
mxfp4_global, None, hf_config=hf_config
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
mixed = {
|
||||
**mxfp4_global,
|
||||
"layer_quant_config": {
|
||||
"layers.0.attn.wkv": {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"block_size": [32, 32],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
assert (
|
||||
DeepseekV4FP8Config.override_quantization_method(
|
||||
mixed, None, hf_config=hf_config
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="DeepGEMM requires CUDA")
|
||||
@pytest.mark.parametrize("scale_dtype", [torch.uint8, torch.float8_e8m0fnu])
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -909,6 +909,184 @@ def test_quant_method_dispatch_target(case):
|
||||
assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls)
|
||||
|
||||
|
||||
def test_quant_method_dispatch_mxfp8_2d_block(default_vllm_config):
|
||||
"""Shape B (DeepSeek-V4.1): 32x32 per-block e8m0 routes to QuarkOCP_MX.
|
||||
|
||||
Verifies that get_quant_method and get_quant_method_target agree (the
|
||||
short-circuit that previously caused them to diverge has been removed).
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
mxfp8_spec = {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"block_size": [32, 32],
|
||||
"symmetric": True,
|
||||
"is_dynamic": False,
|
||||
"scale_type": "float8_e8m0fnu",
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"symmetric": True,
|
||||
"is_dynamic": True,
|
||||
},
|
||||
}
|
||||
config = QuarkConfig(
|
||||
{
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp4",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
},
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {"layers.0.attn.wkv": mxfp8_spec},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
# get_quant_method_target now routes MXFP8 through the matcher chain.
|
||||
wk, ak, mcls = config.get_quant_method_target("layers.0.attn.wkv", LinearBase)
|
||||
assert wk == kMxfp8Static
|
||||
assert mcls is QuarkLinearMethod
|
||||
|
||||
linear = TestLinear()
|
||||
method = config.get_quant_method(linear, "layers.0.attn.wkv")
|
||||
assert isinstance(method, QuarkLinearMethod)
|
||||
assert isinstance(linear.scheme, QuarkOCP_MX)
|
||||
assert linear.scheme.weight_quant_key == kMxfp8Static
|
||||
assert linear.scheme.scale_block_rows == 32
|
||||
|
||||
# Experts still fall through to the global MXFP4 spec.
|
||||
assert (
|
||||
config.get_quant_method_target("layers.0.ffn.experts", RoutedExperts)[2]
|
||||
is not QuarkLinearMethod
|
||||
)
|
||||
|
||||
# 128x128 per-block FP8 (DeepSeek V4) keeps its existing Quark scheme.
|
||||
v4_spec = {
|
||||
"weight": {**mxfp8_spec["weight"], "block_size": [128, 128]},
|
||||
"input_tensors": {**mxfp8_spec["input_tensors"], "group_size": 128},
|
||||
}
|
||||
v4_config = _make_qtensor_config(v4_spec["weight"], v4_spec["input_tensors"])
|
||||
assert v4_config.get_quant_method_target("linear", LinearBase)[2] is (
|
||||
QuarkLinearMethod
|
||||
)
|
||||
linear = TestLinear()
|
||||
assert isinstance(v4_config.get_quant_method(linear, "linear"), QuarkLinearMethod)
|
||||
assert isinstance(linear.scheme, QuarkW8A8Fp8PerBlock)
|
||||
|
||||
|
||||
def test_quant_method_dispatch_mxfp8_canonical(default_vllm_config):
|
||||
"""Shape A (canonical MX): per_group/group_size=32/e8m0 FP8 routes
|
||||
to QuarkOCP_MX with scale_block_rows == 1.
|
||||
|
||||
This covers the canonical MXFP8 spelling (no 2-D block). Validated by
|
||||
synthetic dispatch test only — no canonical-MXFP8 checkpoint on hand.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkOCP_MX
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Static
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
mxfp8_canonical_spec = {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
}
|
||||
config = QuarkConfig(
|
||||
{
|
||||
"global_quant_config": mxfp8_canonical_spec,
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
|
||||
wk, ak, mcls = config.get_quant_method_target("linear", LinearBase)
|
||||
assert wk == kMxfp8Static
|
||||
assert mcls is QuarkLinearMethod
|
||||
|
||||
class TestLinear(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
linear = TestLinear()
|
||||
method = config.get_quant_method(linear, "linear")
|
||||
assert isinstance(method, QuarkLinearMethod)
|
||||
assert isinstance(linear.scheme, QuarkOCP_MX)
|
||||
assert linear.scheme.weight_quant_key == kMxfp8Static
|
||||
assert linear.scheme.scale_block_rows == 1
|
||||
|
||||
|
||||
def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config):
|
||||
"""MXFP8 in a MoE config raises ValueError — experts are unsupported."""
|
||||
from vllm.model_executor.layers.quantization.quark.quark_moe import (
|
||||
QuarkOCP_MX_MoEMethod,
|
||||
)
|
||||
|
||||
default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
mxfp8_spec = {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": False,
|
||||
},
|
||||
"input_tensors": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_group",
|
||||
"group_size": 32,
|
||||
"scale_format": "e8m0",
|
||||
"is_dynamic": True,
|
||||
},
|
||||
}
|
||||
config = QuarkConfig(
|
||||
{
|
||||
"global_quant_config": mxfp8_spec,
|
||||
"layer_type_quant_config": {},
|
||||
"layer_quant_config": {},
|
||||
"exclude": [],
|
||||
}
|
||||
)
|
||||
wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts)
|
||||
assert mcls is QuarkOCP_MX_MoEMethod
|
||||
# The OCP MX MoE constructor should fail loudly for MXFP8.
|
||||
fake_moe_config = MagicMock()
|
||||
with pytest.raises(ValueError, match="MXFP8 experts are not supported"):
|
||||
QuarkOCP_MX_MoEMethod(fake_moe_config, wk, ak)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("weight", "input_tensors"),
|
||||
[
|
||||
@@ -2313,3 +2491,107 @@ def test_quark_int4_canonicalizes_pack_for_kernel_layout(pack_method, symmetric)
|
||||
)
|
||||
|
||||
assert torch.equal(actual, expected)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# override_quantization_method: Quark always delegates to QuarkConfig
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_QUARK_MXFP4_CFG = {
|
||||
"quant_method": "quark",
|
||||
"global_quant_config": {
|
||||
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32},
|
||||
},
|
||||
}
|
||||
|
||||
_QUARK_MXFP8_CFG = {
|
||||
"quant_method": "quark",
|
||||
"global_quant_config": {
|
||||
"weight": {
|
||||
"dtype": "fp8_e4m3",
|
||||
"qscheme": "per_block",
|
||||
"block_size": [32, 32],
|
||||
"scale_type": "float8_e8m0fnu",
|
||||
"symmetric": True,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_QUARK_MIXED_CFG = {
|
||||
"quant_method": "quark",
|
||||
"global_quant_config": {
|
||||
"weight": {"dtype": "fp4", "qscheme": "per_group", "group_size": 32},
|
||||
},
|
||||
"layer_quant_config": {"layers.0.attn": {"weight": {"dtype": "fp8_e4m3"}}},
|
||||
}
|
||||
|
||||
_FP8_CFG = {"quant_method": "fp8"}
|
||||
|
||||
_DSV4_FP8_CFG = {"quant_method": "deepseek_v4_fp8"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"quant_config_cls",
|
||||
[
|
||||
"deepseek_v41",
|
||||
"deepseek_v4",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"hf_quant_cfg, expect_none",
|
||||
[
|
||||
(_QUARK_MXFP4_CFG, True),
|
||||
(_QUARK_MXFP8_CFG, True),
|
||||
(_QUARK_MIXED_CFG, True),
|
||||
(_FP8_CFG, False),
|
||||
(_DSV4_FP8_CFG, False),
|
||||
],
|
||||
ids=[
|
||||
"quark-mxfp4",
|
||||
"quark-mxfp8",
|
||||
"quark-mixed",
|
||||
"native-fp8",
|
||||
"deepseek-v4-fp8",
|
||||
],
|
||||
)
|
||||
def test_quark_override_delegates_to_quark_config(
|
||||
quant_config_cls, hf_quant_cfg, expect_none
|
||||
):
|
||||
"""Any quant_method=='quark' must return None (delegate to QuarkConfig).
|
||||
|
||||
Native FP8 configs should still be claimed by the model-specific config.
|
||||
"""
|
||||
import importlib
|
||||
|
||||
if quant_config_cls == "deepseek_v41":
|
||||
# Module was renamed from deepseek_v4_1 to deepseek_v41 upstream.
|
||||
for mod_name in (
|
||||
"vllm.models.deepseek_v41.quant_config",
|
||||
"vllm.models.deepseek_v4_1.quant_config",
|
||||
):
|
||||
try:
|
||||
mod = importlib.import_module(mod_name)
|
||||
break
|
||||
except ModuleNotFoundError:
|
||||
continue
|
||||
else:
|
||||
pytest.skip("deepseek_v41 quant_config not found")
|
||||
model_type = "deepseek_v41"
|
||||
else:
|
||||
mod = importlib.import_module("vllm.models.deepseek_v4.quant_config")
|
||||
model_type = "deepseek_v4"
|
||||
|
||||
config_cls = mod.DeepseekV4FP8Config
|
||||
hf_config = SimpleNamespace(model_type=model_type)
|
||||
result = config_cls.override_quantization_method(
|
||||
hf_quant_cfg, None, hf_config=hf_config
|
||||
)
|
||||
|
||||
if expect_none:
|
||||
assert result is None, (
|
||||
f"Quark config should delegate to QuarkConfig (return None), got {result!r}"
|
||||
)
|
||||
else:
|
||||
assert result == "deepseek_v4_fp8", (
|
||||
f"Native FP8 should be claimed by DeepseekV4FP8Config, got {result!r}"
|
||||
)
|
||||
|
||||
@@ -45,6 +45,8 @@ from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
ocp_mx_weight_dtype_and_rows,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -158,6 +160,20 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
self.quant_config = quant_config_with_hf_to_vllm_mapper
|
||||
|
||||
def _scale_block_rows(self, prefix: str, layer_type: type[torch.nn.Module]) -> int:
|
||||
"""How many weight rows one checkpoint scale row covers.
|
||||
|
||||
1 for the canonical 1-D per-group MX spelling; ``block_size[0]`` for
|
||||
MXFP8 exports that spell the scale 2-D (DeepSeek-V4.1 uses 32). The
|
||||
scheme cannot work this out itself: it has no handle on the layer name,
|
||||
and tensor-parallel sharding narrows the loaded scale so the ratio is
|
||||
not recoverable from shapes either.
|
||||
"""
|
||||
config = self._find_matched_config(prefix, layer_type)
|
||||
weight = self._unwrap_single_quant_config(config.get("weight"))
|
||||
normalized = ocp_mx_weight_dtype_and_rows(weight)
|
||||
return normalized[1] if normalized is not None else 1
|
||||
|
||||
def get_quant_method(
|
||||
self, layer: torch.nn.Module, prefix: str
|
||||
) -> "QuantizeMethodBase | None":
|
||||
@@ -175,6 +191,11 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls,
|
||||
weight_quant_key=weight_quant_key,
|
||||
activation_quant_key=activation_quant_key,
|
||||
scale_block_rows=(
|
||||
self._scale_block_rows(prefix, type(layer))
|
||||
if scheme_cls is QuarkOCP_MX
|
||||
else 1
|
||||
),
|
||||
weight_config=self._find_matched_config(prefix, type(layer)).get(
|
||||
"weight"
|
||||
),
|
||||
@@ -640,52 +661,18 @@ class QuarkConfig(QuantizationConfig):
|
||||
The rationale for checking only the weight type is that
|
||||
the model loading concept and process primarily concerns the weights themselves.
|
||||
"""
|
||||
# Confirm weights quantized.
|
||||
if not isinstance(weight_quant, dict):
|
||||
# Confirm weights quantized, and normalize the two OCP MX config
|
||||
# dialects (1-D per_group, and MXFP8's 2-D per_block) to a dtype.
|
||||
normalized = ocp_mx_weight_dtype_and_rows(weight_quant)
|
||||
if normalized is None:
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP_MX format: "
|
||||
"weight_quant is not a dictionary."
|
||||
"Quark model's weight quantization is incompatible with OCP MX "
|
||||
"format: %s",
|
||||
weight_quant,
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight qscheme needs to be per group.
|
||||
if weight_quant.get("qscheme") != "per_group":
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"weight is not per_group."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight group size needs to be 32.
|
||||
if weight_quant.get("group_size") != 32:
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"group_size of weight is not 32."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Activations and weight scales need to be in e8m0 format.
|
||||
if weight_quant.get("scale_format") != "e8m0":
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"scale_format of weight is not e8m0."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
# Input and weight dtypes need to be any of fp4,
|
||||
# fp6_e3m2 or fp6_e3m2, possibly mixed.
|
||||
if weight_quant.get("dtype") not in {
|
||||
"fp4",
|
||||
"fp6_e3m2",
|
||||
"fp6_e2m3",
|
||||
}:
|
||||
logger.debug(
|
||||
"Quark model's weight quantization is incompatible with OCP MX format: "
|
||||
"dtype is not in {fp4, fp6_e3m2, fp6_e2m3}."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
|
||||
weight_dtype = weight_quant["dtype"].replace("fp", "mxfp")
|
||||
weight_dtype, _ = normalized
|
||||
weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype]
|
||||
if input_quant is None:
|
||||
activation_quant_key = None
|
||||
@@ -703,6 +690,14 @@ class QuarkConfig(QuantizationConfig):
|
||||
"input scales."
|
||||
)
|
||||
return QuantKeyMatch(False, None, None)
|
||||
elif (
|
||||
input_quant["dtype"] == "fp8_e4m3"
|
||||
and input_quant.get("qscheme") == "per_group"
|
||||
and input_quant.get("group_size") == OCP_MX_BLOCK_SIZE
|
||||
):
|
||||
# MXFP8 activations. Must be checked before the per-tensor FP8
|
||||
# branch below, which would otherwise claim them.
|
||||
activation_quant_key = _ACTIVATION_QUANT_KEY_MAP["mxfp8_e4m3"]
|
||||
elif input_quant["dtype"] == "fp8_e4m3":
|
||||
activation_quant_key = kFp8DynamicTensorSym
|
||||
else:
|
||||
@@ -903,6 +898,7 @@ class QuarkConfig(QuantizationConfig):
|
||||
scheme_cls: type["QuarkScheme"],
|
||||
weight_quant_key: QuantKey | None,
|
||||
activation_quant_key: QuantKey | None,
|
||||
scale_block_rows: int = 1,
|
||||
weight_config: dict[str, Any] | None = None,
|
||||
) -> "QuarkScheme":
|
||||
"""Construct a Quark scheme selected by get_scheme_cls."""
|
||||
@@ -942,6 +938,9 @@ class QuarkConfig(QuantizationConfig):
|
||||
):
|
||||
kwargs["weight_quant_key"] = weight_quant_key
|
||||
|
||||
if scheme_cls is QuarkOCP_MX:
|
||||
kwargs["scale_block_rows"] = scale_block_rows
|
||||
|
||||
scheme = scheme_cls(**kwargs)
|
||||
|
||||
# Raise error if device does not support the scheme
|
||||
|
||||
@@ -1587,7 +1587,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
raise ValueError(
|
||||
f"Unsupported OCP MX dtype combination for MoE: "
|
||||
f"input_dtype={self.input_dtype}, weight_dtype={self.weight_dtype}. "
|
||||
f"Please check that the combination is supported in OCP_MX_Scheme."
|
||||
f"MXFP8 experts are not supported through the MoE path; only "
|
||||
f"the linear path handles MXFP8. Supported MoE weight dtypes "
|
||||
f"are mxfp4, mxfp6_e3m2, and mxfp6_e2m3."
|
||||
)
|
||||
|
||||
# TODO(bowenbao): refactor and introduce backends for other OCP MX schemes,
|
||||
@@ -1676,10 +1678,16 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
if quant_dtype == "mxfp4":
|
||||
assert dim % 2 == 0
|
||||
return dim // 2
|
||||
else:
|
||||
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
|
||||
# FP6 packs 4 * 6 = 24 bits on 3 bytes.
|
||||
assert (dim * 3) % 4 == 0
|
||||
return (dim * 3) // 4
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"Unsupported quant_dtype in QuarkOCP_MX_MoEMethod."
|
||||
f"get_packed_dim: {quant_dtype}. MXFP8 is not supported for "
|
||||
f"MoE experts."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
|
||||
@@ -10,22 +10,31 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
MxFp4LinearKernel,
|
||||
MxFp6LinearKernel,
|
||||
Mxfp8LinearKernel,
|
||||
init_mxfp4_linear_kernel,
|
||||
init_mxfp6_linear_kernel,
|
||||
init_mxfp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
_ACTIVATION_QUANT_KEY_MAP,
|
||||
_WEIGHT_QUANT_KEY_MAP,
|
||||
OCP_MX_BLOCK_SIZE,
|
||||
OCP_MX_UNPACKED_DTYPES,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Static,
|
||||
kMxfp8Static,
|
||||
)
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
ModelWeightParameter,
|
||||
PackedvLLMParameter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -36,7 +45,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class QuarkOCP_MX(QuarkScheme):
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
|
||||
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel | Mxfp8LinearKernel
|
||||
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
|
||||
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
|
||||
|
||||
@@ -44,9 +53,10 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
self,
|
||||
weight_quant_key: QuantKey,
|
||||
activation_quant_key: QuantKey | None,
|
||||
scale_block_rows: int = 1,
|
||||
):
|
||||
super().__init__(weight_quant_key, activation_quant_key)
|
||||
|
||||
self.scale_block_rows = scale_block_rows
|
||||
self.weight_dtype = next(
|
||||
dtype
|
||||
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
|
||||
@@ -62,11 +72,21 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
else None
|
||||
)
|
||||
|
||||
if self.weight_dtype == "mxfp4":
|
||||
self.packed_factor: int | Fraction = 2
|
||||
self.is_unpacked = self.weight_dtype in OCP_MX_UNPACKED_DTYPES
|
||||
|
||||
if self.is_unpacked:
|
||||
self.packed_factor: int | Fraction = 1
|
||||
elif self.weight_dtype == "mxfp4":
|
||||
self.packed_factor = 2
|
||||
else:
|
||||
self.packed_factor = Fraction(numerator=8, denominator=6)
|
||||
|
||||
if self.is_unpacked:
|
||||
# MXFP8 runs on the shared kernel abstraction, which picks a native
|
||||
# backend where one exists and emulates otherwise. The MXFP4/MXFP6
|
||||
# warnings below do not apply.
|
||||
return
|
||||
|
||||
if not current_platform.supports_mx():
|
||||
logger.warning_once(
|
||||
"The current platform does not support native MXFP4/MXFP6 "
|
||||
@@ -88,7 +108,9 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
)
|
||||
|
||||
def get_packed_dim(self, dim: int, quant_dtype: str):
|
||||
if quant_dtype == "mxfp4":
|
||||
if quant_dtype in OCP_MX_UNPACKED_DTYPES:
|
||||
return dim
|
||||
elif quant_dtype == "mxfp4":
|
||||
assert dim % 2 == 0
|
||||
return dim // 2
|
||||
elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
|
||||
@@ -102,6 +124,26 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"open an issue."
|
||||
)
|
||||
|
||||
def _expanding_scale_loader(self, weight_loader: Callable) -> Callable:
|
||||
"""Expand a 2-D block scale to one row per weight row.
|
||||
|
||||
A ``block_size=[R, 32]`` export carries one e8m0 value per R weight
|
||||
rows; the kernels want one per row. At ``R == 1`` the layouts already
|
||||
coincide, so the loader is passed through untouched.
|
||||
"""
|
||||
if self.scale_block_rows == 1:
|
||||
return weight_loader
|
||||
|
||||
rows = self.scale_block_rows
|
||||
|
||||
def expanding_loader(param, loaded_weight, *args, **kwargs):
|
||||
loaded_weight = loaded_weight.view(torch.uint8).repeat_interleave(
|
||||
rows, dim=0
|
||||
)
|
||||
return weight_loader(param, loaded_weight, *args, **kwargs)
|
||||
|
||||
return expanding_loader
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 70
|
||||
@@ -130,36 +172,68 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
"tensor-parallel sharding for this layer."
|
||||
)
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
if self.is_unpacked:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
layer.input_size_per_partition = input_size_per_partition
|
||||
layer.output_size_per_partition = output_size_per_partition
|
||||
layer.params_dtype = params_dtype
|
||||
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition,
|
||||
dtype=MXFP8_VALUE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=self._expanding_scale_loader(weight_loader),
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
layer.weight_block_size = [1, OCP_MX_BLOCK_SIZE]
|
||||
else:
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
# WEIGHT
|
||||
weight = PackedvLLMParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
self.get_packed_dim(input_size_per_partition, self.weight_dtype),
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
packed_dim=1,
|
||||
packed_factor=self.packed_factor,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
# WEIGHT SCALE
|
||||
weight_scale = GroupQuantScaleParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
input_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=torch.uint8,
|
||||
),
|
||||
input_dim=1,
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
if self.weight_quant_key == kMxfp4Static:
|
||||
self.ocp_mx_linear = init_mxfp4_linear_kernel(
|
||||
@@ -170,6 +244,16 @@ class QuarkOCP_MX(QuarkScheme):
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
)
|
||||
elif self.weight_quant_key == kMxfp8Static:
|
||||
self.ocp_mx_linear = init_mxfp8_linear_kernel(
|
||||
weight_shape=layer.weight.shape
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"No OCP MX linear kernel for weight_quant_key="
|
||||
f"{self.weight_quant_key}. Something is wrong, please open an "
|
||||
"issue."
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -44,7 +44,12 @@ def get_quark_ocp_mx_group_size(
|
||||
return None
|
||||
|
||||
assert weight_quant is not None
|
||||
return int(weight_quant["group_size"])
|
||||
group_size = weight_quant.get("group_size")
|
||||
if group_size is None:
|
||||
block_size = weight_quant.get("block_size")
|
||||
if isinstance(block_size, list | tuple) and len(block_size) >= 2:
|
||||
group_size = block_size[1]
|
||||
return int(group_size) if group_size is not None else None
|
||||
|
||||
|
||||
def is_shared_expert_quant_fse_compatible(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -11,6 +12,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp6E2M3Static,
|
||||
kMxfp6E3M2Dynamic,
|
||||
kMxfp6E3M2Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -21,14 +24,19 @@ _WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Static,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Static,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Static,
|
||||
"mxfp8_e4m3": kMxfp8Static,
|
||||
}
|
||||
|
||||
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
|
||||
"mxfp4": kMxfp4Dynamic,
|
||||
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
|
||||
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
|
||||
"mxfp8_e4m3": kMxfp8Dynamic,
|
||||
}
|
||||
|
||||
# Unpacked OCP MX dtypes: one byte per element, no `packed_factor`.
|
||||
OCP_MX_UNPACKED_DTYPES = {"mxfp8_e4m3"}
|
||||
|
||||
OCP_MX_DTYPES = {
|
||||
"mxfp4",
|
||||
"mxfp6_e3m2",
|
||||
@@ -39,6 +47,56 @@ OCP_MX_DTYPES = {
|
||||
}
|
||||
|
||||
|
||||
def ocp_mx_weight_dtype_and_rows(
|
||||
weight_quant: dict[str, Any] | None,
|
||||
) -> tuple[str, int] | None:
|
||||
"""Normalize a Quark weight quant config to ``(mx_dtype, scale_block_rows)``.
|
||||
|
||||
Quark spells OCP MX weights two ways. The canonical one is 1-D per-group:
|
||||
``qscheme="per_group"``, ``group_size=32``, ``scale_format="e8m0"``. MXFP8
|
||||
checkpoints may instead use a 2-D per-block spelling
|
||||
(``qscheme="per_block"``, ``block_size=[R, 32]``,
|
||||
``scale_type="float8_e8m0fnu"``), which carries one scale per ``R`` weight
|
||||
rows rather than one per row. ``scale_block_rows`` is that ``R``; it is 1
|
||||
for the canonical spelling, where the two layouts coincide.
|
||||
|
||||
Returns ``None`` when the config is not an OCP MX weight quantization.
|
||||
"""
|
||||
if not isinstance(weight_quant, dict):
|
||||
return None
|
||||
|
||||
dtype = weight_quant.get("dtype")
|
||||
if not isinstance(dtype, str):
|
||||
return None
|
||||
mx_dtype = dtype.replace("fp", "mxfp")
|
||||
if mx_dtype not in _WEIGHT_QUANT_KEY_MAP:
|
||||
return None
|
||||
|
||||
qscheme = weight_quant.get("qscheme")
|
||||
if qscheme == "per_group":
|
||||
if weight_quant.get("group_size") != OCP_MX_BLOCK_SIZE:
|
||||
return None
|
||||
if weight_quant.get("scale_format") != "e8m0":
|
||||
return None
|
||||
return mx_dtype, 1
|
||||
|
||||
if qscheme == "per_block":
|
||||
# Only the unpacked dtypes are accepted here: expanding a 2-D scale
|
||||
# over a sub-byte packed weight has no checkpoint to validate against.
|
||||
if mx_dtype not in OCP_MX_UNPACKED_DTYPES:
|
||||
return None
|
||||
block_size = list(weight_quant.get("block_size") or [])
|
||||
if len(block_size) != 2 or block_size[1] != OCP_MX_BLOCK_SIZE:
|
||||
return None
|
||||
if weight_quant.get("scale_type") != "float8_e8m0fnu":
|
||||
return None
|
||||
if weight_quant.get("symmetric") is not True or weight_quant.get("is_dynamic"):
|
||||
return None
|
||||
return mx_dtype, block_size[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class OCP_MX_Scheme(str, Enum):
|
||||
w_mxfp4 = "w_mxfp4"
|
||||
w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4"
|
||||
|
||||
@@ -135,15 +135,15 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
# Quark checkpoints are always handled by QuarkConfig.
|
||||
if (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and hf_quant_cfg.get("quant_method") == "quark"
|
||||
):
|
||||
return None
|
||||
if not (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and (
|
||||
hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
or (
|
||||
hf_quant_cfg.get("quant_method") == "quark"
|
||||
and cls._is_quark_mxfp4_ocp(hf_quant_cfg)
|
||||
)
|
||||
)
|
||||
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
):
|
||||
return None
|
||||
model_type = getattr(hf_config, "model_type", None)
|
||||
|
||||
@@ -895,6 +895,11 @@ def _make_deepseek_v4_weights_mapper(
|
||||
# renames the engram fp8 table but not its scale; route the
|
||||
# scale explicitly to the same module.
|
||||
re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv",
|
||||
# Quark exports spell the same tensor ``embed.weight_scale``,
|
||||
# which the ``\.scale$`` rules never match.
|
||||
re.compile(
|
||||
r"(engram\.embed)\.weight_scale$"
|
||||
): r"\1_tokens.weight_scale_inv",
|
||||
re.compile(r"\.scale$"): f".{linear_scale_name}",
|
||||
}
|
||||
else:
|
||||
@@ -908,6 +913,11 @@ def _make_deepseek_v4_weights_mapper(
|
||||
): r"\1.weight_scale_inv",
|
||||
# Same engram reroute as the fp4 branch above.
|
||||
re.compile(r"(engram\.embed)\.scale$"): r"\1_tokens.weight_scale_inv",
|
||||
# Quark exports spell the same tensor ``embed.weight_scale``,
|
||||
# which the ``\.scale$`` rules never match.
|
||||
re.compile(
|
||||
r"(engram\.embed)\.weight_scale$"
|
||||
): r"\1_tokens.weight_scale_inv",
|
||||
re.compile(r"\.scale$"): f".{linear_scale_name}",
|
||||
}
|
||||
return WeightsMapper(
|
||||
|
||||
@@ -79,20 +79,22 @@ class DeepseekV4VLImagePixelInputs(TensorSchema):
|
||||
types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})]
|
||||
|
||||
|
||||
_VL_PREFIX_MAPPING: dict[str, str | None] = {
|
||||
"layers.": "language_model.model.layers.",
|
||||
"embed.": "language_model.model.embed.",
|
||||
"norm.": "language_model.model.norm.",
|
||||
"hc_head": "language_model.model.hc_head",
|
||||
"mtp.": "language_model.model.mtp.",
|
||||
}
|
||||
|
||||
|
||||
def _make_deepseek_v4_vl_weights_mapper(
|
||||
expert_dtype: str, linear_scale_name: str
|
||||
) -> WeightsMapper:
|
||||
"""Text-checkpoint mapping rules re-rooted under ``language_model.``."""
|
||||
base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name)
|
||||
orig_to_new_prefix: dict[str, str | None] = {
|
||||
"layers.": "language_model.model.layers.",
|
||||
"embed.": "language_model.model.embed.",
|
||||
"norm.": "language_model.model.norm.",
|
||||
"hc_head": "language_model.model.hc_head",
|
||||
"mtp.": "language_model.model.mtp.",
|
||||
}
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix=orig_to_new_prefix,
|
||||
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
|
||||
orig_to_new_regex=base.orig_to_new_regex,
|
||||
orig_to_new_suffix={
|
||||
"head.weight": "language_model.lm_head.weight",
|
||||
@@ -131,6 +133,22 @@ class DeepseekV41ForCausalLM(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
|
||||
# Both of these are read off the *class* by
|
||||
# ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs,
|
||||
# to translate a quantization config's checkpoint-style module names into
|
||||
# runtime prefixes. ``__init__`` later overwrites the instance
|
||||
# ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the
|
||||
# prefix rules matter here. Without the class attributes, per-layer specs
|
||||
# naming ``layers.N.attn.wq_a`` never match this wrapper's
|
||||
# ``language_model.model.layers.N.attn.wq_a`` and silently fall back to
|
||||
# the global spec.
|
||||
packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix=dict(_VL_PREFIX_MAPPING),
|
||||
orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"},
|
||||
)
|
||||
|
||||
# The MoE router needs raw token ids to detect image-span tokens
|
||||
# (all carrying image_token_id, see common/mm_preprocess.py) and apply
|
||||
# bias_vl.
|
||||
|
||||
@@ -139,15 +139,17 @@ class DeepseekV4FP8Config(Fp8Config):
|
||||
def override_quantization_method(
|
||||
cls, hf_quant_cfg, user_quant, hf_config=None
|
||||
) -> QuantizationMethods | None:
|
||||
# Quark checkpoints are always handled by QuarkConfig, which knows
|
||||
# how to dispatch each per-layer scheme (MXFP4, MXFP8, etc.).
|
||||
# ``from_config`` below rewrites the config into a single global FP8
|
||||
# scheme, which would discard per-layer specs — so never claim Quark.
|
||||
if isinstance(hf_quant_cfg, dict) and (
|
||||
hf_quant_cfg.get("quant_method") == "quark"
|
||||
):
|
||||
return None
|
||||
if not (
|
||||
isinstance(hf_quant_cfg, dict)
|
||||
and (
|
||||
hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
or (
|
||||
hf_quant_cfg.get("quant_method") == "quark"
|
||||
and cls._is_quark_mxfp4_ocp(hf_quant_cfg)
|
||||
)
|
||||
)
|
||||
and hf_quant_cfg.get("quant_method") in ("fp8", "deepseek_v4_fp8")
|
||||
):
|
||||
return None
|
||||
model_type = getattr(hf_config, "model_type", None)
|
||||
|
||||
Reference in New Issue
Block a user