[Quantization][Refactor][1/N] Adopt QuantKey in QuarkConfig and methods, relying on weight_quant_key, act_quant_key for quant method dispatch (#52958)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
fxmarty-amd
2026-09-01 02:55:54 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent d0e695a91b
commit 63988f3c2d
13 changed files with 1574 additions and 476 deletions
+8 -4
View File
@@ -35,6 +35,10 @@ from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
kMxfp4Static,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import PlatformEnum, current_platform
@@ -346,9 +350,9 @@ def test_online_mxfp4_moe_matches_quark(
online_layer = make_layer("online_layer")
checkpoint_method = QuarkOCP_MX_MoEMethod(
weight_config={"qscheme": "per_group", "dtype": "fp4"},
input_config={"dtype": "fp4", "is_dynamic": True},
moe=checkpoint_layer.moe_config,
weight_quant_key=kMxfp4Static,
activation_quant_key=kMxfp4Dynamic,
)
online_method = Mxfp4OnlineMoEMethod(layer=online_layer)
@@ -566,8 +570,8 @@ def test_online_mxfp4_dense_matches_quark(
checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight)
checkpoint_scheme = QuarkOCP_MX(
weight_quant_spec={"qscheme": "per_group", "dtype": "fp4"},
input_quant_spec={"dtype": "fp4", "is_dynamic": True},
weight_quant_key=kMxfp4Static,
activation_quant_key=kMxfp4Dynamic,
)
checkpoint_scheme.create_weights(
layer=checkpoint_layer,
+792 -67
View File
@@ -10,6 +10,7 @@ See also `tests/kernels/moe/test_ocp_mx_moe.py`.
import importlib.metadata
from dataclasses import dataclass
from importlib.util import find_spec
from types import SimpleNamespace
import huggingface_hub
import lm_eval
@@ -18,22 +19,62 @@ import torch
from packaging import version
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.model_executor.layers.fused_moe import (
RoutedExperts,
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
QuarkConfig,
QuarkLinearMethod,
QuarkNVFP4,
QuarkOCP_MX,
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
)
from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501
QuarkMoEMethod,
QuarkW4A8Fp8MoEMethod,
QuarkW8A8Int8MoEMethod,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
is_layer_skipped,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@@ -62,6 +103,576 @@ AITER_AVAILABLE = is_aiter_found_and_supported()
DEVICE_TYPE = current_platform.device_type
@dataclass(frozen=True)
class QTensorConfig:
name: str
weight: QuarkQTensorHint
input_tensors: QuarkQTensorHint
weight_quant_key: QuantKey | None = None
act_quant_key: QuantKey | None = None
dispatch_cls: type[QuarkScheme] | type[QuarkMoEMethod] | None = None
expected_error: tuple[type[Exception], str] | None = None
QTENSOR_CONFIGS = [
QTensorConfig(
name="fp8_w8a8_static_tensor",
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
},
weight_quant_key=kFp8StaticTensorSym,
act_quant_key=kFp8StaticTensorSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_static_tensor_single_entry_lists",
weight=[
{
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
}
],
input_tensors=[
{
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
}
],
weight_quant_key=kFp8StaticTensorSym,
act_quant_key=kFp8StaticTensorSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_dynamic_tensor",
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": True,
},
weight_quant_key=kFp8StaticTensorSym,
act_quant_key=kFp8DynamicTensorSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_dynamic_token",
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_channel",
"is_dynamic": True,
},
weight_quant_key=kFp8StaticChannelSym,
act_quant_key=kFp8DynamicTokenSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_channel_static_tensor",
weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
},
weight_quant_key=kFp8StaticChannelSym,
act_quant_key=kFp8StaticTensorSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_tensor_dynamic_token",
weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_channel",
"is_dynamic": True,
},
weight_quant_key=kFp8StaticTensorSym,
act_quant_key=kFp8DynamicTokenSym,
dispatch_cls=QuarkW8A8Fp8,
),
QTensorConfig(
name="fp8_w8a8_dynamic_block_fp32",
weight={
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
},
weight_quant_key=kFp8Static128BlockSym,
act_quant_key=kFp8Dynamic128Sym,
dispatch_cls=QuarkW8A8Fp8PerBlock,
),
QTensorConfig(
name="fp8_w8a8_dynamic_block_e8m0",
weight={
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
"scale_type": "float8_e8m0fnu",
},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
},
weight_quant_key=kFp8Static128BlockE8M0Sym,
act_quant_key=kFp8Dynamic128Sym,
dispatch_cls=QuarkW8A8Fp8PerBlock,
),
QTensorConfig(
name="fp8_w8a8_block_static_input",
weight={
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": False,
"group_size": 128,
"symmetric": True,
},
expected_error=(NotImplementedError, "No quark compatible scheme"),
),
QTensorConfig(
name="fp8_w8a8_block_group_size_mismatch",
weight={
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 64,
"symmetric": True,
},
expected_error=(NotImplementedError, "No quark compatible scheme"),
),
QTensorConfig(
name="fp8_w8a8_block_missing_block_size",
weight={
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
},
expected_error=(ValueError, "requires `block_size`"),
),
QTensorConfig(
name="int8_w8a8_static_symmetric",
weight={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
weight_quant_key=kInt8StaticTensorSym,
act_quant_key=kInt8StaticTensorSym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_static_asymmetric",
weight={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": False,
},
weight_quant_key=kInt8StaticTensorSym,
act_quant_key=kInt8StaticTensorAsym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_channel_static_symmetric",
weight={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
weight_quant_key=kInt8StaticChannelSym,
act_quant_key=kInt8StaticTensorSym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_channel_static_asymmetric",
weight={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": False,
},
weight_quant_key=kInt8StaticChannelSym,
act_quant_key=kInt8StaticTensorAsym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_dynamic_tensor_symmetric",
weight={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": True,
"symmetric": True,
},
weight_quant_key=kInt8StaticTensorSym,
act_quant_key=kInt8DynamicTensorSym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_dynamic_tensor_asymmetric",
weight={
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": True,
"symmetric": False,
},
weight_quant_key=kInt8StaticTensorSym,
act_quant_key=kInt8DynamicTensorAsym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_dynamic_token",
weight={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": True,
"symmetric": True,
},
weight_quant_key=kInt8StaticChannelSym,
act_quant_key=kInt8DynamicTokenSym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="int8_w8a8_dynamic_token_asymmetric",
weight={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
},
input_tensors={
"dtype": "int8",
"qscheme": "per_channel",
"is_dynamic": True,
"symmetric": False,
},
weight_quant_key=kInt8StaticChannelSym,
act_quant_key=kInt8DynamicTokenAsym,
dispatch_cls=QuarkW8A8Int8,
),
QTensorConfig(
name="ocp_mx_mxfp4_weight_only",
weight={
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors=None,
weight_quant_key=kMxfp4Static,
act_quant_key=None,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="ocp_mx_mxfp4_activation",
weight={
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors={
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": True,
},
weight_quant_key=kMxfp4Static,
act_quant_key=kMxfp4Dynamic,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="ocp_mx_mxfp6_e3m2",
weight={
"dtype": "fp6_e3m2",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors={
"dtype": "fp6_e3m2",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": True,
},
weight_quant_key=kMxfp6E3M2Static,
act_quant_key=kMxfp6E3M2Dynamic,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="ocp_mx_mxfp4_mxfp6_e3m2_activation",
weight={
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors={
"dtype": "fp6_e3m2",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": True,
},
weight_quant_key=kMxfp4Static,
act_quant_key=kMxfp6E3M2Dynamic,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="ocp_mx_mxfp4_mxfp6_e2m3_activation",
weight={
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors={
"dtype": "fp6_e2m3",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": True,
},
weight_quant_key=kMxfp4Static,
act_quant_key=kMxfp6E2M3Dynamic,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="ocp_mx_mxfp6_e2m3",
weight={
"dtype": "fp6_e2m3",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
input_tensors={
"dtype": "fp6_e2m3",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": True,
},
weight_quant_key=kMxfp6E2M3Static,
act_quant_key=kMxfp6E2M3Dynamic,
dispatch_cls=QuarkOCP_MX,
),
QTensorConfig(
name="nvfp4",
weight=[
{
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 16,
"is_dynamic": False,
},
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
],
input_tensors=[
{
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 16,
"is_dynamic": True,
},
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
],
weight_quant_key=kNvfp4Static,
act_quant_key=kNvfp4Dynamic,
dispatch_cls=QuarkNVFP4,
),
QTensorConfig(
name="w4a8_fp8_static",
weight=[
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
{
"dtype": "int4",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
"ch_axis": 0,
},
],
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
},
weight_quant_key=kInt4W4A8StaticChannelSym,
act_quant_key=kFp8StaticTensorSym,
dispatch_cls=QuarkW4A8Fp8MoEMethod,
),
QTensorConfig(
name="w4a8_fp8_dynamic",
weight=[
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
{
"dtype": "int4",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
"ch_axis": 0,
},
],
input_tensors={
"dtype": "fp8_e4m3",
"qscheme": "per_channel",
"is_dynamic": True,
},
weight_quant_key=kInt4W4A8StaticChannelSym,
act_quant_key=kFp8DynamicTokenSym,
dispatch_cls=QuarkW4A8Fp8MoEMethod,
),
QTensorConfig(
name="w4a8_fp8_static_single_entry_input",
weight=[
{"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False},
{
"dtype": "int4",
"qscheme": "per_channel",
"is_dynamic": False,
"symmetric": True,
"ch_axis": 0,
},
],
input_tensors=[
{
"dtype": "fp8_e4m3",
"qscheme": "per_tensor",
"is_dynamic": False,
}
],
weight_quant_key=kInt4W4A8StaticChannelSym,
act_quant_key=kFp8StaticTensorSym,
dispatch_cls=QuarkW4A8Fp8MoEMethod,
),
]
def _make_qtensor_config(
weight: QuarkQTensorHint,
input_tensors: QuarkQTensorHint,
exclude: list[str] | None = None,
) -> QuarkConfig:
return QuarkConfig(
{
"global_quant_config": {
"weight": weight,
"input_tensors": input_tensors,
},
"layer_type_quant_config": {},
"exclude": exclude or [],
}
)
def _make_test_moe_config() -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
activation=MoEActivation.SILU,
device=current_platform.device_type,
routing_method=RoutingMethodType.Renormalize,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
)
if QUARK_MXFP4_AVAILABLE:
from quark.torch.export.nn.modules.realquantizer import StaticScaledRealQuantizer
from quark.torch.kernel import mx as mx_kernel
@@ -98,83 +709,197 @@ def test_quark_config_preserves_existing_packed_modules_mapping():
assert config.packed_modules_mapping["custom_proj"] == ["a", "b"]
def test_quark_fp8_w8a8_detects_per_block_config():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
def test_quant_method_dispatch_ignored(default_vllm_config):
config = _make_qtensor_config(None, None, exclude=["linear", "experts"])
assert config._is_fp8_w8a8(weight_config, input_config)
class TestLinear(LinearBase):
def __init__(self):
torch.nn.Module.__init__(self)
class TestRoutedExperts(RoutedExperts):
def __init__(self):
torch.nn.Module.__init__(self)
self.moe_config = _make_test_moe_config()
assert config.get_quant_method_target("linear", LinearBase) == (
None,
None,
UnquantizedLinearMethod,
)
assert isinstance(
config.get_quant_method(TestLinear(), "linear"), UnquantizedLinearMethod
)
assert config.get_quant_method_target("experts", RoutedExperts) == (
None,
None,
UnquantizedFusedMoEMethod,
)
assert isinstance(
config.get_quant_method(TestRoutedExperts(), "experts"),
UnquantizedFusedMoEMethod,
)
dynamic_mxfp4_config = _make_qtensor_config(
{
"dtype": "fp4",
"qscheme": "per_group",
"group_size": 32,
"scale_format": "e8m0",
"is_dynamic": False,
},
None,
exclude=["self_attn.q_proj", "mlp.down_proj"],
)
dynamic_mxfp4_config.dynamic_mxfp4_quant = True
assert dynamic_mxfp4_config.get_quant_method_target(
"self_attn.q_proj", LinearBase
) == (kMxfp4Static, None, QuarkLinearMethod)
attention_proj = TestLinear()
assert isinstance(
dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"),
QuarkLinearMethod,
)
assert isinstance(attention_proj.scheme, QuarkOCP_MX)
assert attention_proj.scheme.dynamic_mxfp4_quant
assert dynamic_mxfp4_config.get_quant_method_target(
"mlp.down_proj", LinearBase
) == (None, None, UnquantizedLinearMethod)
assert isinstance(
dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"),
UnquantizedLinearMethod,
)
def test_quark_fp8_w8a8_rejects_per_block_static_input():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": False,
"group_size": 128,
"symmetric": True,
}
@pytest.mark.parametrize("case", QTENSOR_CONFIGS, ids=lambda case: case.name)
def test_quant_method_dispatch_target(case):
config = _make_qtensor_config(case.weight, case.input_tensors)
if case.expected_error is not None:
error_type, error_message = case.expected_error
with pytest.raises(error_type, match=error_message):
config.get_quant_method_target("linear", LinearBase)
return
assert not config._is_fp8_w8a8(weight_config, input_config)
assert case.dispatch_cls is not None
is_linear = issubclass(case.dispatch_cls, QuarkScheme)
weight_quant_key, act_quant_key, method_cls = config.get_quant_method_target(
"linear" if is_linear else "experts",
LinearBase if is_linear else RoutedExperts,
)
assert weight_quant_key == case.weight_quant_key
assert act_quant_key == case.act_quant_key
assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls)
def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch():
config = QuarkConfig({})
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"block_size": [128, 128],
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 64,
"symmetric": True,
}
@pytest.mark.parametrize(
("weight", "input_tensors"),
[
pytest.param(
{
"dtype": "int8",
"qscheme": "per_group",
"is_dynamic": False,
"symmetric": True,
},
{
"dtype": "int8",
"qscheme": "per_tensor",
"is_dynamic": False,
"symmetric": True,
},
id="single_entry",
),
pytest.param(
[
{"dtype": "int8", "qscheme": "per_tensor"},
{"dtype": "int8", "qscheme": "per_tensor"},
],
[
{"dtype": "int8", "qscheme": "per_tensor"},
{"dtype": "int8", "qscheme": "per_tensor"},
],
id="multi_entry",
),
],
)
def test_quant_method_dispatch_unsupported(weight, input_tensors):
config = _make_qtensor_config(weight, input_tensors)
assert not config._is_fp8_w8a8(weight_config, input_config)
class TestRoutedExperts(RoutedExperts):
def __init__(self):
torch.nn.Module.__init__(self)
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
config.get_quant_method_target("experts", RoutedExperts)
with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"):
config.get_quant_method(TestRoutedExperts(), "experts")
def test_quark_w8a8_fp8_per_block_requires_block_size():
weight_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_block",
"is_dynamic": False,
"symmetric": True,
}
input_config = {
"dtype": "fp8_e4m3",
"qscheme": "per_group",
"is_dynamic": True,
"group_size": 128,
"symmetric": True,
}
@pytest.mark.parametrize(
"case",
[case for case in QTENSOR_CONFIGS if case.expected_error is None],
ids=lambda case: case.name,
)
def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_config):
config = _make_qtensor_config(case.weight, case.input_tensors)
assert case.dispatch_cls is not None
if issubclass(case.dispatch_cls, QuarkScheme):
with pytest.raises(ValueError, match="requires `block_size`"):
QuarkW8A8Fp8PerBlock(weight_config, input_config)
class TestLinear(LinearBase):
def __init__(self):
torch.nn.Module.__init__(self)
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.quark.schemes."
"quark_w8a8_fp8.get_current_vllm_config",
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
)
layer = TestLinear()
method = config.get_quant_method(layer, "linear")
assert isinstance(method, QuarkLinearMethod)
assert isinstance(layer.scheme, case.dispatch_cls)
if case.weight_quant_key == kFp8Static128BlockE8M0Sym:
# TODO: Remove once E8M0 quant key is properly handled in oracle
assert layer.scheme.weight_quant_key == kFp8Static128BlockSym
else:
assert layer.scheme.weight_quant_key == case.weight_quant_key
assert layer.scheme.activation_quant_key == case.act_quant_key
else:
class TestRoutedExperts(RoutedExperts):
def __init__(self):
torch.nn.Module.__init__(self)
self.moe_config = _make_test_moe_config()
for target in (
"select_fp8_moe_backend",
"select_int8_moe_backend",
"select_mxfp4_moe_backend",
"backend_to_kernel_cls",
"select_nvfp4_moe_backend",
):
monkeypatch.setattr(
f"vllm.model_executor.layers.quantization.quark.quark_moe.{target}",
lambda *args, **kwargs: (object(), object()),
)
# AssertionError: W4A8 FP8 MoE requires ROCm AITER fused MoE support
monkeypatch.setattr(
"vllm.model_executor.layers.quantization.quark.quark_moe."
"rocm_aiter_ops.is_fused_moe_enabled",
lambda: True,
)
layer = TestRoutedExperts()
method = config.get_quant_method(layer, "experts")
assert isinstance(method, case.dispatch_cls)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import fnmatch
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, NamedTuple, cast
import torch
from transformers import PretrainedConfig
@@ -37,9 +37,35 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
QuarkW8A8Int8,
)
from vllm.model_executor.layers.quantization.quark.utils import (
QuarkQTensorHint,
deep_compare,
should_ignore_layer,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
kMxfp4Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
@@ -55,6 +81,15 @@ logger = init_logger(__name__)
_DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"})
class QuantKeyMatch(NamedTuple):
matches: bool
activation_quant_key: QuantKey | None
weight_quant_key: QuantKey | None
def __bool__(self) -> bool:
return self.matches
class QuarkConfig(QuantizationConfig):
def __init__(
self,
@@ -155,41 +190,102 @@ class QuarkConfig(QuantizationConfig):
def get_quant_method(
self, layer: torch.nn.Module, prefix: str
) -> "QuantizeMethodBase | None":
# Check if the layer is skipped for quantization.
weight_quant_key, activation_quant_key, method_cls = (
self.get_quant_method_target(prefix, type(layer))
)
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
if should_ignore_layer(
is_ignored = should_ignore_layer(
prefix,
ignore=exclude_layers,
fused_mapping=self.packed_modules_mapping,
check_children=isinstance(layer, RoutedExperts),
):
if isinstance(layer, RoutedExperts):
return UnquantizedFusedMoEMethod(layer.moe_config)
if (
"self_attn" not in prefix # only quantize attention projections
or not getattr(self, "dynamic_mxfp4_quant", False)
or not isinstance(layer, LinearBase) # Ignore other methods
):
return UnquantizedLinearMethod()
)
dynamic_mxfp4_quant = (
is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant
)
scheme = self.get_scheme(
layer=layer,
layer_name=prefix,
dynamic_mxfp4_quant=True,
if method_cls is UnquantizedFusedMoEMethod:
return UnquantizedFusedMoEMethod(layer.moe_config)
if method_cls is UnquantizedLinearMethod:
return UnquantizedLinearMethod()
if method_cls is QuarkLinearMethod:
_, _, scheme_cls = self.get_scheme_cls(type(layer), prefix)
scheme = self.init_scheme(
scheme_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
dynamic_mxfp4_quant=dynamic_mxfp4_quant,
)
layer.scheme = scheme
return QuarkLinearMethod(self)
if isinstance(layer, LinearBase):
scheme = self.get_scheme(layer=layer, layer_name=prefix)
layer.scheme = scheme
return QuarkLinearMethod(self)
if isinstance(layer, Attention):
if method_cls is QuarkKVCacheMethod:
return QuarkKVCacheMethod(self)
if method_cls is not None and issubclass(method_cls, QuarkMoEMethod):
return QuarkMoEMethod.get_moe_method(
self,
module=layer,
method_cls=method_cls,
weight_quant_key=weight_quant_key,
activation_quant_key=activation_quant_key,
)
if isinstance(layer, RoutedExperts):
return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix)
return None
def get_quant_method_target(
self, prefix: str, layer_type: type[torch.nn.Module]
) -> tuple[
QuantKey | None,
QuantKey | None,
type[QuantizeMethodBase]
| type[UnquantizedLinearMethod]
| type[UnquantizedFusedMoEMethod]
| None,
]:
"""Return weight key, activation key, and quant method class for
the given ``prefix`` and ``layer_type``.
This is the counterpart of ``get_quant_method`` without quant method
instantiation.
TODO: integrate as part of base QuantizationConfig.get_quant_method_target
in the future.
``None`` denotes an unquantized/BF16 activation.
"""
is_routed_experts = issubclass(layer_type, RoutedExperts)
if should_ignore_layer(
prefix,
ignore=cast(list[str], self.quant_config.get("exclude")),
fused_mapping=self.packed_modules_mapping,
check_children=is_routed_experts,
):
if is_routed_experts:
return None, None, UnquantizedFusedMoEMethod
if issubclass(layer_type, LinearBase):
if "self_attn" in prefix and self.dynamic_mxfp4_quant:
weight_quant_key, activation_key, _ = self.get_scheme_cls(
layer_type, prefix
)
return weight_quant_key, activation_key, QuarkLinearMethod
return None, None, UnquantizedLinearMethod
return None, None, None
if issubclass(layer_type, LinearBase):
weight_quant_key, activation_key, _ = self.get_scheme_cls(
layer_type, prefix
)
return weight_quant_key, activation_key, QuarkLinearMethod
if issubclass(layer_type, Attention):
return None, None, QuarkKVCacheMethod
if is_routed_experts:
weight_quant_key, activation_quant_key, quant_method_cls = (
QuarkMoEMethod.get_moe_method_target(self, layer_type, prefix)
)
if quant_method_cls is None:
return None, None, None
return weight_quant_key, activation_quant_key, quant_method_cls
return None, None, None
@classmethod
def from_config(cls, config: dict[str, Any]) -> "QuarkConfig":
export_config = config.get("export")
@@ -290,15 +386,20 @@ class QuarkConfig(QuantizationConfig):
def _is_fp8_w4a8(
self,
weight_quant: list[dict[str, Any]] | None,
input_quant: dict[str, Any] | None,
) -> bool:
weight_quant: QuarkQTensorHint,
input_quant: QuarkQTensorHint,
) -> QuantKeyMatch:
# Confirm weights and input quantized.
if weight_quant is None or input_quant is None:
return False
if not isinstance(weight_quant, list) or len(weight_quant) != 2:
return False
if isinstance(input_quant, list):
if len(input_quant) != 1:
return QuantKeyMatch(False, None, None)
input_quant = input_quant[0]
if (
not isinstance(weight_quant, list)
or len(weight_quant) != 2
or not isinstance(input_quant, dict)
):
return QuantKeyMatch(False, None, None)
# Confirm weight scheme is supported
is_w4a8_dtype = (
@@ -321,24 +422,26 @@ class QuarkConfig(QuantizationConfig):
and is_static_weight
and is_per_tensor_fp8_and_per_channel_int4_weight
):
return False
return QuantKeyMatch(False, None, None)
# Dynamic quantization is always supported if weights supported.
if input_quant.get("is_dynamic"):
return True
return QuantKeyMatch(True, kFp8DynamicTokenSym, kInt4W4A8StaticChannelSym)
# Confirm activation scheme is supported.
is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor"
return is_per_tensor_activation
if not is_per_tensor_activation:
return QuantKeyMatch(False, None, None)
return QuantKeyMatch(True, kFp8StaticTensorSym, kInt4W4A8StaticChannelSym)
def _is_fp8_w8a8(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> bool:
) -> QuantKeyMatch:
# Confirm weights and input quantized.
if weight_quant is None or input_quant is None:
return False
if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict):
return QuantKeyMatch(False, None, None)
# Confirm weight scheme is supported
is_fp8_dtype = (
@@ -358,64 +461,125 @@ class QuarkConfig(QuantizationConfig):
)
if not is_fp8_dtype or not is_static_weight:
return False
return QuantKeyMatch(False, None, None)
if is_per_block_weight:
if not input_quant.get("is_dynamic"):
return False
return (
return QuantKeyMatch(False, None, None)
matches = (
input_quant.get("qscheme") == "per_group"
and input_quant.get("group_size") == block_size[1]
and input_quant.get("symmetric") is True
)
if not matches:
return QuantKeyMatch(False, None, None)
weight_quant_key = (
kFp8Static128BlockE8M0Sym
if weight_quant.get("scale_type") == "float8_e8m0fnu"
else kFp8Static128BlockSym
)
return QuantKeyMatch(True, kFp8Dynamic128Sym, weight_quant_key)
if not is_per_tensor_or_channel_weight:
return False
return QuantKeyMatch(False, None, None)
# Dynamic quantization is always supported if tensor/channel weights
# are supported.
if input_quant.get("is_dynamic"):
return True
activation_quant_key = (
kFp8DynamicTokenSym
if input_quant.get("qscheme") == "per_channel"
else kFp8DynamicTensorSym
)
weight_quant_key = (
kFp8StaticChannelSym
if weight_quant.get("qscheme") == "per_channel"
else kFp8StaticTensorSym
)
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
# Confirm activation scheme is supported.
is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor"
return is_per_tensor_activation
if not is_per_tensor_activation:
return QuantKeyMatch(False, None, None)
weight_quant_key = (
kFp8StaticChannelSym
if weight_quant.get("qscheme") == "per_channel"
else kFp8StaticTensorSym
)
return QuantKeyMatch(True, kFp8StaticTensorSym, weight_quant_key)
def _is_static_tensor_w8a8(
def _is_w8a8_int8(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> bool:
) -> QuantKeyMatch:
# Confirm weights and input quantized.
if weight_quant is None or input_quant is None:
return False
if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict):
return QuantKeyMatch(False, None, None)
is_int8_dtype = (
weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8"
)
is_tensor = (
weight_quant.get("qscheme") in ["per_tensor", "per_channel"]
and input_quant.get("qscheme") == "per_tensor"
is_valid_weight_scheme = weight_quant.get("qscheme") in [
"per_tensor",
"per_channel",
]
is_static_input = input_quant.get(
"qscheme"
) == "per_tensor" and not input_quant.get("is_dynamic")
is_dynamic_input = (
input_quant.get("qscheme") == "per_channel"
and input_quant.get("is_dynamic") is True
)
is_static = not weight_quant.get("is_dynamic") and not input_quant.get(
"is_dynamic"
)
is_weight_symmetric = weight_quant.get("symmetric") is True
# Both symmetric and asymmetric input quantization supported.
# Only symmetric weight quantization supported.
return is_int8_dtype and is_tensor and is_weight_symmetric and is_static
matches = (
is_int8_dtype
and is_valid_weight_scheme
and is_weight_symmetric
and (
(is_static_input and not weight_quant.get("is_dynamic"))
or is_dynamic_input
)
)
if not matches:
return QuantKeyMatch(False, None, None)
weight_quant_key = (
kInt8StaticChannelSym
if weight_quant.get("qscheme") == "per_channel"
else kInt8StaticTensorSym
)
if is_static_input:
activation_quant_key = (
kInt8StaticTensorSym
if input_quant.get("symmetric") is True
else kInt8StaticTensorAsym
)
elif weight_quant.get("qscheme") == "per_channel":
activation_quant_key = (
kInt8DynamicTokenSym
if input_quant.get("symmetric") is True
else kInt8DynamicTokenAsym
)
else:
activation_quant_key = (
kInt8DynamicTensorSym
if input_quant.get("symmetric") is True
else kInt8DynamicTensorAsym
)
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
def _is_w4a8_mxfp4_fp8(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> bool:
) -> QuantKeyMatch:
if weight_quant is None or input_quant is None:
return False
return QuantKeyMatch(False, None, None)
is_weight_mxfp4 = (
weight_quant.get("dtype") == "fp4"
@@ -432,53 +596,24 @@ class QuarkConfig(QuantizationConfig):
and input_quant.get("symmetric") is True # Symmetric quantization
)
return is_weight_mxfp4 and is_input_fp8
def _is_dynamic_per_token_w8a8(
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
) -> bool:
"""Detect W8A8 INT8 with per-tensor or per-channel
weights and dynamic per-token input."""
if weight_quant is None or input_quant is None:
return False
is_int8_dtype = (
weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8"
)
is_valid_weight_scheme = weight_quant.get("qscheme") in [
"per_tensor",
"per_channel",
]
is_per_token_input = input_quant.get("qscheme") == "per_channel"
is_dynamic_input = input_quant.get("is_dynamic") is True
is_weight_symmetric = weight_quant.get("symmetric") is True
return (
is_int8_dtype
and is_valid_weight_scheme
and is_per_token_input
and is_dynamic_input
and is_weight_symmetric
)
if not (is_weight_mxfp4 and is_input_fp8):
return QuantKeyMatch(False, None, None)
return QuantKeyMatch(True, kFp8StaticTensorSym, kMxfp4Static)
def _is_nvfp4(
self,
weight_quant: dict[str, Any] | list[dict[str, Any]] | None,
input_quant: dict[str, Any] | list[dict[str, Any]] | None,
) -> bool:
weight_quant: QuarkQTensorHint,
input_quant: QuarkQTensorHint,
) -> QuantKeyMatch:
# Confirm weights and input quantized.
if weight_quant is None or input_quant is None:
return False
return QuantKeyMatch(False, None, None)
# Confirm both weight_quant and input_quant are lists with 2 elements
if not isinstance(weight_quant, list) or len(weight_quant) != 2:
return False
return QuantKeyMatch(False, None, None)
if not isinstance(input_quant, list) or len(input_quant) != 2:
return False
return QuantKeyMatch(False, None, None)
# First element should be fp4 with per_group quantization
is_fp4_per_group_weight = (
@@ -506,36 +641,35 @@ class QuarkConfig(QuantizationConfig):
and not input_quant[1].get("is_dynamic")
)
return (
matches = (
is_fp4_per_group_weight # type: ignore[return-value]
and is_fp4_per_group_input
and is_fp8_per_tensor_weight
and is_fp8_per_tensor_input
)
if not matches:
return QuantKeyMatch(False, None, None)
return QuantKeyMatch(True, kNvfp4Dynamic, kNvfp4Static)
def _is_w_ocp_mx_a_x(
self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None
) -> bool:
self,
weight_quant: dict[str, Any] | None,
input_quant: dict[str, Any] | None,
allow_static_fp8: bool = False,
) -> QuantKeyMatch:
"""
This check returns True only if it is an OCP-MX weight quantization.
The activation can be any data type (e.g., FP16/BF16, FP8, or OCP-MX format).
The activation can be FP16/BF16, OCP MXFP4, MXFP8, FP8.
The rationale for checking only the weight type is that
the model loading concept and process primarily concerns the weights themselves.
"""
# Confirm weights quantized.
if weight_quant is None:
if not isinstance(weight_quant, dict):
logger.debug(
"Quark model's weight quantization is incompatible with OCP_MX format: "
"weight_quant is not set."
"weight_quant is not a dictionary."
)
return False
if isinstance(weight_quant, list):
logger.debug(
"Quark model's weight quantization is incompatible with OCP_MX format: "
"weight_quant is a list (e.g. fp8_w4a8), OCP_MX requires a single dict."
)
return False
return QuantKeyMatch(False, None, None)
# Input and weight qscheme needs to be per group.
if weight_quant.get("qscheme") != "per_group":
@@ -543,7 +677,7 @@ class QuarkConfig(QuantizationConfig):
"Quark model's weight quantization is incompatible with OCP MX format: "
"weight is not per_group."
)
return False
return QuantKeyMatch(False, None, None)
# Input and weight group size needs to be 32.
if weight_quant.get("group_size") != 32:
@@ -551,7 +685,7 @@ class QuarkConfig(QuantizationConfig):
"Quark model's weight quantization is incompatible with OCP MX format: "
"group_size of weight is not 32."
)
return False
return QuantKeyMatch(False, None, None)
# Activations and weight scales need to be in e8m0 format.
if weight_quant.get("scale_format") != "e8m0":
@@ -559,7 +693,7 @@ class QuarkConfig(QuantizationConfig):
"Quark model's weight quantization is incompatible with OCP MX format: "
"scale_format of weight is not e8m0."
)
return False
return QuantKeyMatch(False, None, None)
# Input and weight dtypes need to be any of fp4,
# fp6_e3m2 or fp6_e3m2, possibly mixed.
@@ -572,9 +706,48 @@ class QuarkConfig(QuantizationConfig):
"Quark model's weight quantization is incompatible with OCP MX format: "
"dtype is not in {fp4, fp6_e3m2, fp6_e2m3}."
)
return False
return QuantKeyMatch(False, None, None)
return True
weight_dtype = weight_quant["dtype"].replace("fp", "mxfp")
weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype]
if input_quant is None:
activation_quant_key = None
elif not input_quant.get("is_dynamic"):
if (
allow_static_fp8
and input_quant.get("dtype") == "fp8_e4m3"
and input_quant.get("qscheme") == "per_tensor"
and input_quant.get("symmetric") is True
):
activation_quant_key = kFp8StaticTensorSym
else:
logger.debug(
"Quark model's OCP MX quantization is incompatible with static "
"input scales."
)
return QuantKeyMatch(False, None, None)
elif input_quant["dtype"] == "fp8_e4m3":
activation_quant_key = kFp8DynamicTensorSym
else:
input_dtype = input_quant["dtype"].replace("fp", "mxfp")
if input_dtype not in _ACTIVATION_QUANT_KEY_MAP:
raise ValueError(
f"Unsupported input_dtype={input_dtype} in Quark's vLLM "
"integration. Supported activation dtypes are "
f"{_ACTIVATION_QUANT_KEY_MAP.keys()}, or None for "
"weight-only quantization."
)
activation_quant_key = _ACTIVATION_QUANT_KEY_MAP[input_dtype]
return QuantKeyMatch(True, activation_quant_key, weight_quant_key)
@staticmethod
def _unwrap_single_quant_config(
quant_config: QuarkQTensorHint,
) -> dict[str, Any] | None:
"""Unwrap one-entry quantization-config lists."""
if isinstance(quant_config, list):
return quant_config[0] if len(quant_config) == 1 else None
return quant_config
def get_layer_quant_config_from_name(
self, layer_name: str
@@ -617,14 +790,14 @@ class QuarkConfig(QuantizationConfig):
return None
def _find_matched_config(
self, layer_name: str, module: torch.nn.Module
self, layer_name: str, module: torch.nn.Module | type[torch.nn.Module]
) -> dict[str, Any]:
# Priority order:
# 1. layer_quant_config,
# 2. layer_type_quant_config,
# 3. global_quant_config.
layer_type = cast(str, type(module))
layer_type = cast(str, module if isinstance(module, type) else type(module))
layer_type_quant_config = cast(
dict[str, Any], self.quant_config.get("layer_type_quant_config")
)
@@ -662,51 +835,71 @@ class QuarkConfig(QuantizationConfig):
return layer_quant_config
return fallback_config
def _get_scheme_from_config(
self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False
) -> "QuarkScheme":
def _get_scheme_cls_from_config(
self, config: dict[str, Any]
) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]:
if config.get("output_tensors") or config.get("bias"):
raise NotImplementedError(
"Currently, Quark models with output_tensors "
"and bias quantized are not supported"
)
weight_config = cast(dict[str, Any], config.get("weight"))
input_config = cast(dict[str, Any], config.get("input_tensors"))
weight_config = cast(QuarkQTensorHint, config.get("weight"))
input_config = cast(QuarkQTensorHint, config.get("input_tensors"))
if self._is_nvfp4(weight_config, input_config):
return QuarkNVFP4()
elif self._is_fp8_w8a8(weight_config, input_config):
is_fp8_w8a8_supported = self._check_scheme_supported(
QuarkW8A8Fp8.get_min_capability(), error=False
if (
isinstance(weight_config, list)
and len(weight_config) > 1
or isinstance(input_config, list)
and len(input_config) > 1
):
if match := self._is_nvfp4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkNVFP4,
)
raise NotImplementedError(
"Multi-entry weight or activation quantization configs are only "
"supported for NVFP4."
)
if is_fp8_w8a8_supported:
if weight_config.get("qscheme") == "per_block":
return QuarkW8A8Fp8PerBlock(weight_config, input_config)
return QuarkW8A8Fp8(weight_config, input_config)
elif self._is_static_tensor_w8a8(weight_config, input_config):
weight_qscheme = cast(str, weight_config.get("qscheme"))
return QuarkW8A8Int8(
qscheme=weight_qscheme,
is_static_input_scheme=True,
input_symmetric=input_config.get("symmetric"),
weight_config = self._unwrap_single_quant_config(weight_config)
input_config = self._unwrap_single_quant_config(input_config)
if (
isinstance(weight_config, dict)
and weight_config.get("qscheme") == "per_block"
and not weight_config.get("block_size")
):
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization requires "
"`block_size` in the weight quantization config."
)
elif self._is_w4a8_mxfp4_fp8(weight_config, input_config):
is_w4a8_supported = self._check_scheme_supported(
QuarkW4A8_MXFP4_FP8.get_min_capability(), error=False
if match := self._is_fp8_w8a8(weight_config, input_config):
assert weight_config is not None
if weight_config.get("qscheme") == "per_block":
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW8A8Fp8PerBlock,
)
return match.weight_quant_key, match.activation_quant_key, QuarkW8A8Fp8
elif match := self._is_w8a8_int8(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW8A8Int8,
)
if is_w4a8_supported:
return QuarkW4A8_MXFP4_FP8(weight_config, input_config)
elif self._is_dynamic_per_token_w8a8(weight_config, input_config):
weight_qscheme = cast(str, weight_config.get("qscheme"))
return QuarkW8A8Int8(
qscheme=weight_qscheme,
is_static_input_scheme=False,
input_symmetric=input_config.get("symmetric"),
)
elif self._is_w_ocp_mx_a_x(weight_config, input_config):
return QuarkOCP_MX(
weight_config, input_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant
elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A8_MXFP4_FP8,
)
elif match := self._is_w_ocp_mx_a_x(weight_config, input_config):
return match.weight_quant_key, match.activation_quant_key, QuarkOCP_MX
raise NotImplementedError(
"No quark compatible scheme was found. "
@@ -714,19 +907,51 @@ class QuarkConfig(QuantizationConfig):
f"Input config: {input_config}"
)
def get_scheme(
self, layer: torch.nn.Module, layer_name: str, dynamic_mxfp4_quant: bool = False
) -> "QuarkScheme":
layer_quant_config = self._find_matched_config(layer_name, layer)
# Find the quant_scheme
scheme = self._get_scheme_from_config(
layer_quant_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant
def get_scheme_cls(
self, layer_type: type[torch.nn.Module], layer_name: str
) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]:
"""Return quantization keys and scheme class without initializing it."""
return self._get_scheme_cls_from_config(
self._find_matched_config(layer_name, layer_type)
)
def init_scheme(
self,
scheme_cls: type["QuarkScheme"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
) -> "QuarkScheme":
"""Construct a Quark scheme selected by get_scheme_cls."""
if scheme_cls not in (
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
QuarkOCP_MX,
QuarkNVFP4,
QuarkW4A8_MXFP4_FP8,
):
raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}")
kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key}
# These classes handle different possible `weight_quant_key`.
if scheme_cls in (
QuarkW8A8Fp8,
QuarkW8A8Fp8PerBlock,
QuarkW8A8Int8,
QuarkOCP_MX,
):
kwargs["weight_quant_key"] = weight_quant_key
if scheme_cls is QuarkOCP_MX:
kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant
scheme = scheme_cls(**kwargs)
# Raise error if device does not support the scheme
# (e.g. fp8 needs ada lovelace)
self._check_scheme_supported(scheme.get_min_capability())
return scheme
@staticmethod
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from typing import TYPE_CHECKING
import torch
@@ -56,19 +56,27 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
make_nvfp4_moe_quant_config,
select_nvfp4_moe_backend,
)
from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
OCP_MX_Scheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4W4A8StaticChannelSym,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
kMxfp4Dynamic,
kNvfp4Dynamic,
@@ -82,6 +90,9 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
if TYPE_CHECKING:
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
logger = init_logger(__name__)
__all__ = [
@@ -93,67 +104,154 @@ __all__ = [
class QuarkMoEMethod(FusedMoEMethodBase):
def __init__(self, moe: FusedMoEConfig):
supported_activation_quant_keys: list[QuantKey | None] = []
supported_weight_quant_keys: list[QuantKey] = []
def __init__(
self,
moe: FusedMoEConfig,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
if activation_quant_key not in self.supported_activation_quant_keys:
raise ValueError(
f"Unsupported activation quant key: {activation_quant_key}"
)
if weight_quant_key not in self.supported_weight_quant_keys:
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
self.activation_quant_key = activation_quant_key
self.weight_quant_key = weight_quant_key
self.has_bias = self.moe.has_bias
@staticmethod
def get_moe_method(
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
module: RoutedExperts,
def get_moe_method_target(
quant_config: "QuarkConfig",
layer_type: type[torch.nn.Module],
layer_name: str,
) -> "QuarkMoEMethod":
layer_quant_config = quant_config._find_matched_config(layer_name, module)
) -> tuple[
QuantKey | None,
QuantKey | None,
type["QuarkMoEMethod"] | None,
]:
"""Return the quantization target for a routed-experts layer."""
layer_quant_config = quant_config._find_matched_config(layer_name, layer_type)
if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"):
raise NotImplementedError(
"Currently, Quark models with "
"output_tensors and bias "
"quantized are not supported"
)
weight_config: QuarkQTensorHint = layer_quant_config.get("weight")
input_config: QuarkQTensorHint = layer_quant_config.get("input_tensors")
weight_config = layer_quant_config.get("weight")
input_config = layer_quant_config.get("input_tensors")
if (
isinstance(weight_config, list)
and len(weight_config) > 1
or isinstance(input_config, list)
and len(input_config) > 1
):
if match := quant_config._is_fp8_w4a8(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW4A8Fp8MoEMethod,
)
if match := quant_config._is_nvfp4(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkNvfp4MoEMethod,
)
raise RuntimeError("Unsupported FusedMoe scheme")
if quant_config._is_fp8_w4a8(weight_config, input_config):
return QuarkW4A8Fp8MoEMethod(weight_config, input_config, module.moe_config)
elif quant_config._is_nvfp4(weight_config, input_config):
return QuarkNvfp4MoEMethod(
weight_config, input_config, module.moe_config, quant_config
weight_config = quant_config._unwrap_single_quant_config(weight_config)
input_config = quant_config._unwrap_single_quant_config(input_config)
if match := quant_config._is_fp8_w8a8(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW8A8Fp8MoEMethod,
)
elif quant_config._is_fp8_w8a8(weight_config, input_config):
return QuarkW8A8Fp8MoEMethod(weight_config, input_config, module.moe_config)
elif quant_config._is_w_ocp_mx_a_x(weight_config, input_config):
if match := quant_config._is_w_ocp_mx_a_x(
weight_config, input_config, allow_static_fp8=True
):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkOCP_MX_MoEMethod,
)
if match := quant_config._is_w8a8_int8(weight_config, input_config):
return (
match.weight_quant_key,
match.activation_quant_key,
QuarkW8A8Int8MoEMethod,
)
raise RuntimeError("Unsupported FusedMoe scheme")
@staticmethod
def get_moe_method(
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
module: RoutedExperts,
method_cls: type["QuarkMoEMethod"],
weight_quant_key: QuantKey | None,
activation_quant_key: QuantKey | None,
) -> "QuarkMoEMethod":
if weight_quant_key is None:
raise RuntimeError("Unsupported FusedMoe scheme")
if method_cls is QuarkW4A8Fp8MoEMethod:
return QuarkW4A8Fp8MoEMethod(module.moe_config, activation_quant_key)
elif method_cls is QuarkNvfp4MoEMethod:
return QuarkNvfp4MoEMethod(
module.moe_config, quant_config, activation_quant_key
)
elif method_cls is QuarkW8A8Fp8MoEMethod:
return QuarkW8A8Fp8MoEMethod(
module.moe_config, weight_quant_key, activation_quant_key
)
elif method_cls is QuarkOCP_MX_MoEMethod:
# All OCP MX schemes (W4A16, W4A8, etc.) handled by QuarkOCP_MX_MoEMethod
# Backend selection happens inside via oracle
return QuarkOCP_MX_MoEMethod(weight_config, input_config, module.moe_config)
elif quant_config._is_static_tensor_w8a8(
weight_config, input_config
) or quant_config._is_dynamic_per_token_w8a8(weight_config, input_config):
return QuarkOCP_MX_MoEMethod(
module.moe_config, weight_quant_key, activation_quant_key
)
elif method_cls is QuarkW8A8Int8MoEMethod:
return QuarkW8A8Int8MoEMethod(
weight_config, input_config, module.moe_config
module.moe_config, weight_quant_key, activation_quant_key
)
else:
raise RuntimeError("Unsupported FusedMoe scheme")
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [
kFp8StaticTensorSym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
]
supported_weight_quant_keys = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
]
def __init__(
self,
weight_config: dict[str, Any],
input_config: dict[str, Any],
moe: FusedMoEConfig,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
self.weight_quant = weight_config
self.input_quant = input_config
self.weight_qscheme = self.weight_quant.get("qscheme")
self.input_qscheme = self.input_quant.get("qscheme")
self.weight_dtype = self.weight_quant.get("dtype", "").replace(
"fp8_e4m3", "fp8"
super().__init__(moe, weight_quant_key, activation_quant_key)
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.input_dtype = self.input_quant.get("dtype", "").replace("fp8_e4m3", "fp8")
self.input_qscheme = (
"per_channel"
if activation_quant_key == kFp8DynamicTokenSym
else "per_tensor"
)
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
per_tensor = (
self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor"
)
@@ -170,28 +268,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
f"{self.weight_qscheme}, {self.input_qscheme}"
) # noqa E501
self.static_input_scales = not self.input_quant.get("is_dynamic")
if self.static_input_scales and per_channel:
raise ValueError(
"For FP8 Fused MoE layer, we require either per tensor or "
"channelwise, dynamic per token quantization."
)
# Determine quant keys for oracle backend selection
if per_channel:
weight_key = kFp8StaticChannelSym
activation_key = kFp8DynamicTokenSym
elif self.static_input_scales:
weight_key = kFp8StaticTensorSym
activation_key = kFp8StaticTensorSym
else:
weight_key = kFp8StaticTensorSym
activation_key = kFp8DynamicTensorSym
self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
config=moe,
weight_key=weight_key,
activation_key=activation_key,
weight_key=self.weight_quant_key,
activation_key=self.activation_quant_key,
)
self.model_type = getattr(
@@ -500,17 +580,41 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
"""Quark W8A8 INT8 MoE method."""
supported_activation_quant_keys = [
kInt8StaticTensorSym,
kInt8StaticTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTensorAsym,
kInt8DynamicTokenSym,
kInt8DynamicTokenAsym,
]
supported_weight_quant_keys = [
kInt8StaticChannelSym,
kInt8StaticTensorSym,
]
def __init__(
self,
weight_config: dict[str, Any],
input_config: dict[str, Any],
moe: FusedMoEConfig,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
self.weight_quant = weight_config
self.input_quant = input_config
self.weight_qscheme = self.weight_quant.get("qscheme", "per_tensor")
self.static_input_scales = not self.input_quant.get("is_dynamic", False)
super().__init__(moe, weight_quant_key, activation_quant_key)
# TODO: oracle needs to support asym int8.
if self.activation_quant_key == kInt8StaticTensorAsym:
self.activation_quant_key = kInt8StaticTensorSym
if self.activation_quant_key == kInt8DynamicTensorAsym:
self.activation_quant_key = kInt8DynamicTensorSym
if self.activation_quant_key == kInt8DynamicTokenAsym:
self.activation_quant_key = kInt8DynamicTokenSym
self.weight_qscheme = (
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
)
assert self.activation_quant_key is not None
self.static_input_scales = self.activation_quant_key.scale.static
self.moe_quant_config: FusedMoEQuantConfig | None = None
self.moe_kernel: mk.FusedMoEKernel | None = None
@@ -528,17 +632,10 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
# Map the Quark weight scheme to oracle quant keys. Per-channel
# weights pair with dynamic per-token activations; per-tensor
# weights with dynamic per-tensor activations.
if self.weight_qscheme == "per_channel":
weight_key = kInt8StaticChannelSym
activation_key = kInt8DynamicTokenSym
else:
weight_key = kInt8StaticTensorSym
activation_key = kInt8DynamicTensorSym
self.int8_backend, self.experts_cls = select_int8_moe_backend(
config=moe,
weight_key=weight_key,
activation_key=activation_key,
weight_key=self.weight_quant_key,
activation_key=self.activation_quant_key,
)
def create_weights(
@@ -876,15 +973,18 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
]
supported_weight_quant_keys = [kInt4W4A8StaticChannelSym]
def __init__(
self,
weight_config: dict[str, Any],
input_config: dict[str, Any],
moe: FusedMoEConfig,
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
self.weight_quant = weight_config
self.input_quant = input_config
super().__init__(moe, kInt4W4A8StaticChannelSym, activation_quant_key)
assert rocm_aiter_ops.is_fused_moe_enabled(), (
"W4A8 FP8 MoE requires ROCm AITER fused MoE support."
@@ -1032,37 +1132,36 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [
*_ACTIVATION_QUANT_KEY_MAP.values(),
kFp8DynamicTensorSym,
kFp8StaticTensorSym,
None,
]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
def __init__(
self,
weight_config: dict[str, Any],
input_config: dict[str, Any] | None,
moe: FusedMoEConfig,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
self.weight_quant = weight_config
self.input_quant = input_config
weight_qscheme = self.weight_quant.get("qscheme")
if not weight_qscheme == "per_group":
raise ValueError(
"For MX(FP4) Fused MoE layers, only per-group scales "
f"for weights are supported. Found {weight_qscheme}."
) # noqa E501
self.weight_dtype = self.weight_quant["dtype"].replace("fp", "mxfp")
if self.input_quant is not None:
input_quant = self.input_quant["dtype"]
if input_quant in ["fp4", "fp6_e3m2", "fp6_e2m3"]:
self.input_dtype = input_quant.replace("fp", "mxfp")
elif input_quant == "fp8_e4m3":
self.input_dtype = input_quant.replace("fp8_e4m3", "fp8")
else:
raise NotImplementedError(
f"Current input dtype {input_quant} is not compatible \
with OCP MX (weight) MoE quantization. Please open an issue"
)
else:
super().__init__(moe, weight_quant_key, activation_quant_key)
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
if quant_key == weight_quant_key
)
if activation_quant_key in {kFp8DynamicTensorSym, kFp8StaticTensorSym}:
self.input_dtype: str | None = "fp8"
elif activation_quant_key is None:
self.input_dtype = None
else:
self.input_dtype = next(
dtype
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
if quant_key == activation_quant_key
)
self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
self.input_dtype, self.weight_dtype
@@ -1085,10 +1184,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
self.w13_precision_config = None
self.w2_precision_config = None
if self.input_quant is not None:
self.static_input_scales = not self.input_quant.get("is_dynamic")
else:
self.static_input_scales = False
self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
# Select backend based on OCP MX scheme
if self.ocp_mx_scheme == "w_mxfp4":
@@ -1388,6 +1484,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
f"in vLLM for OCP MX scheme {self.ocp_mx_scheme}. Please open an issue."
)
else:
assert self.input_dtype is not None
return ocp_mx_moe_quant_config(
quant_dtype=self.input_dtype,
weight_dtype=self.weight_dtype,
@@ -1459,24 +1556,24 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
class QuarkNvfp4MoEMethod(QuarkMoEMethod):
supported_activation_quant_keys = [kNvfp4Dynamic]
supported_weight_quant_keys = [kNvfp4Static]
def __init__(
self,
weight_config: dict[str, Any],
input_config: dict[str, Any],
moe: FusedMoEConfig,
quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821
activation_quant_key: QuantKey | None,
):
super().__init__(moe)
self.weight_quant = weight_config
self.input_quant = input_config
super().__init__(moe, kNvfp4Static, activation_quant_key)
self.quant_config = quant_config
self.group_size = 16
# Select experts implementation.
self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
config=self.moe,
weight_key=kNvfp4Static,
activation_key=kNvfp4Dynamic,
weight_key=self.weight_quant_key,
activation_key=self.activation_quant_key,
)
def create_weights(
@@ -14,6 +14,11 @@ from vllm.model_executor.kernels.linear.nvfp4.emulation import (
from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import (
QuarkScheme,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
ModelWeightParameter,
@@ -36,9 +41,14 @@ class QuarkNVFP4(QuarkScheme):
- input_scale_2: bfloat16/float32, scalar (global input scale)
"""
supported_activation_quant_keys = [kNvfp4Dynamic]
supported_weight_quant_keys = [kNvfp4Static]
def __init__(
self,
activation_quant_key: QuantKey | None,
):
super().__init__(kNvfp4Static, activation_quant_key)
self.kernel = init_nvfp4_linear_kernel()
self.group_size = 16
@@ -3,7 +3,6 @@
from collections.abc import Callable
from fractions import Fraction
from typing import Any
import torch
@@ -15,15 +14,14 @@ from vllm.model_executor.kernels.linear import (
init_mxfp6_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
_ACTIVATION_QUANT_KEY_MAP,
_WEIGHT_QUANT_KEY_MAP,
OCP_MX_BLOCK_SIZE,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
from vllm.model_executor.parameter import (
@@ -38,45 +36,32 @@ from .quark_scheme import QuarkScheme
logger = init_logger(__name__)
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
}
class QuarkOCP_MX(QuarkScheme):
ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel
supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None]
supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()]
def __init__(
self,
weight_quant_spec: dict[str, Any],
input_quant_spec: dict[str, Any] | None,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
dynamic_mxfp4_quant: bool = False,
):
super().__init__(weight_quant_key, activation_quant_key)
self.dynamic_mxfp4_quant = dynamic_mxfp4_quant
self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp")
self.input_dtype: str | None = None
if input_quant_spec is not None:
self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp")
if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]:
raise ValueError(
f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. "
f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, "
"or None for weight-only quantization."
self.weight_dtype = next(
dtype
for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
if quant_key == weight_quant_key
)
self.input_dtype = (
next(
dtype
for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
if quant_key == activation_quant_key
)
self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype]
self.activation_quant_key = (
_ACTIVATION_QUANT_KEY_MAP[self.input_dtype]
if self.input_dtype is not None
if activation_quant_key is not None
else None
)
@@ -85,17 +70,6 @@ class QuarkOCP_MX(QuarkScheme):
else:
self.packed_factor = Fraction(numerator=8, denominator=6)
if input_quant_spec is None:
self.static_input_scales = False
else:
self.static_input_scales = not input_quant_spec.get("is_dynamic")
if self.static_input_scales:
raise NotImplementedError(
"QuarkOCP_MX with static input scales is currently not "
"implemented. Please open an issue."
)
if not current_platform.supports_mx():
logger.warning_once(
"The current platform does not support native MXFP4/MXFP6 "
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
__all__ = ["QuarkScheme"]
@@ -14,6 +16,23 @@ class QuarkScheme(ABC):
of different quantization schemes supported by Quark.
"""
supported_activation_quant_keys: list[QuantKey | None] = []
supported_weight_quant_keys: list[QuantKey] = []
def __init__(
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
if activation_quant_key not in self.supported_activation_quant_keys:
raise ValueError(
f"Unsupported activation quant key: {activation_quant_key}"
)
if weight_quant_key not in self.supported_weight_quant_keys:
raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
self.activation_quant_key = activation_quant_key
self.weight_quant_key = weight_quant_key
@classmethod
@abstractmethod
def get_min_capability(cls) -> int:
@@ -3,7 +3,6 @@
from collections.abc import Callable
from fractions import Fraction
from typing import Any
import torch
import torch.nn.functional as F
@@ -11,7 +10,10 @@ import torch.nn.functional as F
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
get_fp8_min_max,
kFp8StaticTensorSym,
kMxfp4Static,
)
from vllm.model_executor.parameter import (
GroupQuantScaleParameter,
@@ -38,39 +40,28 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
Uses the AITER Triton kernel and falls back to emulation if AITER not available.
"""
supported_activation_quant_keys = [kFp8StaticTensorSym]
supported_weight_quant_keys = [kMxfp4Static]
def __init__(
self,
weight_quant_spec: dict[str, Any],
input_quant_spec: dict[str, Any],
activation_quant_key: QuantKey | None,
):
super().__init__(kMxfp4Static, activation_quant_key)
self.out_dtype = None
self.weight_dtype = "mxfp4"
self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte
self.weight_block_size = OCP_MX_BLOCK_SIZE
self.is_static_input_scheme = not input_quant_spec.get("is_dynamic")
self.input_qscheme = input_quant_spec.get("qscheme") # "per_tensor"
self.fp8_min, self.fp8_max = get_fp8_min_max()
self.fp8_dtype = current_platform.fp8_dtype()
if not self.is_static_input_scheme:
raise NotImplementedError(
"Dynamic FP8 activation quantization is not yet supported "
"for W4A8. The current implementation expects static per-tensor "
"FP8 scales stored in the checkpoint."
)
kernel_supported_gpu = False
if current_platform.is_rocm():
kernel_supported_gpu = current_platform.supports_mx()
self.use_aiter_kernel = (
is_aiter_found_and_supported()
and self.is_static_input_scheme
and kernel_supported_gpu
)
self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu
if not self.use_aiter_kernel:
logger.warning_once(
@@ -128,17 +119,16 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
layer.register_parameter("weight_scale", weight_scale)
# INPUT SCALE (FP8 per-tensor static scale)
if self.is_static_input_scheme:
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
input_scale = PerTensorScaleParameter(
data=torch.empty(
len(output_partition_sizes),
dtype=torch.float32,
),
weight_loader=weight_loader,
)
# Initialize to avoid NaN
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Ensuring weights & scales are non-trainable
@@ -147,16 +137,15 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme):
layer.weight_scale.data, requires_grad=False
)
if self.is_static_input_scheme:
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
input_scale = layer.input_scale.data
# For fused modules (QKV), take the max scale
if input_scale.numel() != 1:
input_scale = input_scale.max()
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
layer.input_scale = torch.nn.Parameter(
torch.tensor(input_scale, dtype=torch.float32),
requires_grad=False,
)
def apply_weights(
self,
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from typing import Any, cast
import torch
from torch.nn import Parameter
@@ -18,8 +17,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
kFp8Dynamic128Sym,
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8Static128BlockE8M0Sym,
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
@@ -42,36 +44,26 @@ logger = init_logger(__name__)
class QuarkW8A8Fp8(QuarkScheme):
supported_activation_quant_keys: list[QuantKey | None] = [
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticTensorSym,
]
supported_weight_quant_keys: list[QuantKey] = [
kFp8StaticChannelSym,
kFp8StaticTensorSym,
]
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
self.is_static_input_scheme: bool = False
self.input_qscheme: str | None = None
if input_config is not None:
self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic"))
self.input_qscheme = cast(str, input_config.get("qscheme"))
if self.weight_qscheme == "per_block":
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization should use "
"QuarkW8A8Fp8PerBlock."
)
per_token_activation = (
not self.is_static_input_scheme and self.input_qscheme == "per_channel"
)
per_channel_weight = self.weight_qscheme == "per_channel"
self.activation_quant_key = (
kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym
)
# A per-output-channel weight scale is one fp32 value per weight row
# (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the
# canonical compressed-tensors CHANNEL strategy, so kernel selection
# (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly.
self.weight_quant_key = (
kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym
super().__init__(weight_quant_key, activation_quant_key)
self.weight_qscheme = (
"per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor"
)
self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@@ -122,6 +114,8 @@ class QuarkW8A8Fp8(QuarkScheme):
layer.input_scale = Parameter(input_scale, requires_grad=False)
else:
weight_scale = layer.weight_scale.data
assert self.activation_quant_key is not None
if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN:
weight_scale = weight_scale.view(-1, 1)
layer.weight = Parameter(weight.t(), requires_grad=False)
@@ -191,6 +185,7 @@ class QuarkW8A8Fp8(QuarkScheme):
input_scale[:] = torch.finfo(torch.float32).min
layer.register_parameter("input_scale", input_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -210,22 +205,23 @@ class QuarkW8A8Fp8(QuarkScheme):
class QuarkW8A8Fp8PerBlock(QuarkScheme):
supported_activation_quant_keys = [kFp8Dynamic128Sym]
supported_weight_quant_keys = [
kFp8Static128BlockSym,
kFp8Static128BlockE8M0Sym,
]
def __init__(
self, weight_config: dict[str, Any], input_config: dict[str, Any] | None
self,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.weight_qscheme = cast(str, weight_config.get("qscheme"))
if self.weight_qscheme != "per_block":
raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.")
block_size = weight_config.get("block_size")
if not block_size:
raise ValueError(
"Quark W8A8 FP8 per-block weight quantization requires "
"`block_size` in the weight quantization config."
)
self.weight_config = weight_config
self.weight_block_size = list(block_size)
self.activation_quant_key = kFp8Dynamic128Sym
self.weight_quant_key = kFp8Static128BlockSym
super().__init__(weight_quant_key, activation_quant_key)
self.weight_scale_dtype = weight_quant_key.scale.dtype
if self.weight_quant_key == kFp8Static128BlockE8M0Sym:
# TODO: oracle needs to support kFp8Static128BlockE8M0Sym.
self.weight_quant_key = kFp8Static128BlockSym
self.weight_block_size = list(weight_quant_key.scale.group_shape)
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
@@ -272,8 +268,8 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
layer.register_parameter("weight", weight)
scale_dtype = (
torch.float8_e8m0fnu
if self.weight_config.get("scale_type") == "float8_e8m0fnu"
self.weight_scale_dtype
if self.weight_scale_dtype != torch.float32
else None
)
weight_scale = create_fp8_scale_parameter(
@@ -287,6 +283,7 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme):
# DeepSeek V4 weight mappers route checkpoint ".scale" tensors here.
layer.register_parameter("weight_scale_inv", weight_scale)
assert self.activation_quant_key is not None
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
@@ -10,6 +10,16 @@ from vllm.model_executor.kernels.linear import (
init_int8_linear_kernel,
)
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt8DynamicTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTokenAsym,
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
kInt8StaticTensorAsym,
kInt8StaticTensorSym,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
ChannelQuantScaleParameter,
@@ -21,15 +31,32 @@ logger = init_logger(__name__)
class QuarkW8A8Int8(QuarkScheme):
supported_activation_quant_keys = [
kInt8StaticTensorSym,
kInt8StaticTensorAsym,
kInt8DynamicTensorSym,
kInt8DynamicTensorAsym,
kInt8DynamicTokenSym,
kInt8DynamicTokenAsym,
]
supported_weight_quant_keys = [
kInt8StaticChannelSym,
kInt8StaticTensorSym,
]
def __init__(
self,
qscheme: str,
is_static_input_scheme: bool | None,
input_symmetric: bool | None,
weight_quant_key: QuantKey,
activation_quant_key: QuantKey | None,
):
self.qscheme = qscheme
self.is_static_input_scheme = is_static_input_scheme
self.input_symmetric = input_symmetric
super().__init__(weight_quant_key, activation_quant_key)
self.qscheme = (
"per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
)
assert activation_quant_key is not None
self.is_static_input_scheme = activation_quant_key.scale.static
self.input_symmetric = activation_quant_key.symmetric
@classmethod
def get_min_capability(cls) -> int:
@@ -8,6 +8,8 @@ from typing import Any
import regex as re
import torch
QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None
def deep_compare(dict1: Any, dict2: Any) -> bool:
if type(dict1) is not type(dict2):
@@ -3,11 +3,32 @@
from enum import Enum
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
kMxfp6E2M3Dynamic,
kMxfp6E2M3Static,
kMxfp6E3M2Dynamic,
kMxfp6E3M2Static,
)
logger = init_logger(__name__)
OCP_MX_BLOCK_SIZE = 32
_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Static,
"mxfp6_e3m2": kMxfp6E3M2Static,
"mxfp6_e2m3": kMxfp6E2M3Static,
}
_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = {
"mxfp4": kMxfp4Dynamic,
"mxfp6_e3m2": kMxfp6E3M2Dynamic,
"mxfp6_e2m3": kMxfp6E2M3Dynamic,
}
OCP_MX_DTYPES = {
"mxfp4",
"mxfp6_e3m2",
@@ -222,6 +222,11 @@ kFp8Dynamic128Sym = QuantKey(FP8_DTYPE, kDynamic128Scale, symmetric=True)
kStatic128BlockScale = ScaleDesc(torch.float32, True, GroupShape(128, 128))
kFp8Static128BlockSym = QuantKey(FP8_DTYPE, kStatic128BlockScale, symmetric=True)
kFp8Static128BlockE8M0Sym = QuantKey(
FP8_DTYPE,
ScaleDesc(torch.float8_e8m0fnu, True, GroupShape(128, 128)),
symmetric=True,
)
kMxfp8StaticScale = ScaleDesc(torch.uint8, True, GroupShape(1, 32))
kMxfp8Static = QuantKey(FP8_DTYPE, kMxfp8StaticScale, symmetric=True)
@@ -284,6 +289,9 @@ kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)
kInt8StaticTensorSym = QuantKey(torch.int8, kStaticTensorScale, symmetric=True)
kInt8DynamicTensorSym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=True)
kInt8DynamicTokenAsym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=False)
kInt8StaticTensorAsym = QuantKey(torch.int8, kStaticTensorScale, symmetric=False)
kInt8DynamicTensorAsym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=False)
# INT4 W4A8 quantization keys