From 63988f3c2d79db505ca2ac24dc1ed51f4b63249a Mon Sep 17 00:00:00 2001 From: fxmarty-amd Date: Tue, 1 Sep 2026 09:55:54 +0200 Subject: [PATCH] [Quantization][Refactor][1/N] Adopt `QuantKey` in `QuarkConfig` and methods, relying on `weight_quant_key`, `act_quant_key` for quant method dispatch (#52958) Signed-off-by: Felix Marty Co-authored-by: Andreas Karatzas --- tests/quantization/test_online_mxfp4.py | 12 +- tests/quantization/test_quark.py | 859 ++++++++++++++++-- .../layers/quantization/quark/quark.py | 547 +++++++---- .../layers/quantization/quark/quark_moe.py | 321 ++++--- .../quantization/quark/schemes/quark_nvfp4.py | 10 + .../quark/schemes/quark_ocp_mx.py | 62 +- .../quark/schemes/quark_scheme.py | 19 + .../quark/schemes/quark_w4a8_mxfp4_fp8.py | 65 +- .../quark/schemes/quark_w8a8_fp8.py | 85 +- .../quark/schemes/quark_w8a8_int8.py | 39 +- .../layers/quantization/quark/utils.py | 2 + .../layers/quantization/utils/ocp_mx_utils.py | 21 + .../layers/quantization/utils/quant_utils.py | 8 + 13 files changed, 1574 insertions(+), 476 deletions(-) diff --git a/tests/quantization/test_online_mxfp4.py b/tests/quantization/test_online_mxfp4.py index 95c814e9fc..f4112875d4 100644 --- a/tests/quantization/test_online_mxfp4.py +++ b/tests/quantization/test_online_mxfp4.py @@ -35,6 +35,10 @@ from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( mxfp4_quantize, quant_dequant_mxfp4, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kMxfp4Dynamic, + kMxfp4Static, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.utils import replace_parameter from vllm.platforms import PlatformEnum, current_platform @@ -346,9 +350,9 @@ def test_online_mxfp4_moe_matches_quark( online_layer = make_layer("online_layer") checkpoint_method = QuarkOCP_MX_MoEMethod( - weight_config={"qscheme": "per_group", "dtype": "fp4"}, - input_config={"dtype": "fp4", "is_dynamic": True}, moe=checkpoint_layer.moe_config, + weight_quant_key=kMxfp4Static, + activation_quant_key=kMxfp4Dynamic, ) online_method = Mxfp4OnlineMoEMethod(layer=online_layer) @@ -566,8 +570,8 @@ def test_online_mxfp4_dense_matches_quark( checkpoint_weight, checkpoint_weight_scale = mxfp4_quantize(weight) checkpoint_scheme = QuarkOCP_MX( - weight_quant_spec={"qscheme": "per_group", "dtype": "fp4"}, - input_quant_spec={"dtype": "fp4", "is_dynamic": True}, + weight_quant_key=kMxfp4Static, + activation_quant_key=kMxfp4Dynamic, ) checkpoint_scheme.create_weights( layer=checkpoint_layer, diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index c64b1f2d7b..36c1c9411c 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -10,6 +10,7 @@ See also `tests/kernels/moe/test_ocp_mx_moe.py`. import importlib.metadata from dataclasses import dataclass from importlib.util import find_spec +from types import SimpleNamespace import huggingface_hub import lm_eval @@ -18,22 +19,62 @@ import torch from packaging import version from vllm._aiter_ops import is_aiter_found_and_supported +from vllm.model_executor.layers.fused_moe import ( + RoutedExperts, + UnquantizedFusedMoEMethod, +) +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 QuarkConfig, QuarkLinearMethod, + QuarkNVFP4, + QuarkOCP_MX, QuarkW8A8Fp8, QuarkW8A8Fp8PerBlock, QuarkW8A8Int8, ) from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 + QuarkMoEMethod, QuarkW4A8Fp8MoEMethod, QuarkW8A8Int8MoEMethod, ) +from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( quant_dequant_mxfp4, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, is_layer_skipped, + kFp8Dynamic128Sym, + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8Static128BlockE8M0Sym, + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kFp8StaticTensorSym, + kInt4W4A8StaticChannelSym, + kInt8DynamicTensorAsym, + kInt8DynamicTensorSym, + kInt8DynamicTokenAsym, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, + kInt8StaticTensorAsym, + kInt8StaticTensorSym, + kMxfp4Dynamic, + kMxfp4Static, + kMxfp6E2M3Dynamic, + kMxfp6E2M3Static, + kMxfp6E3M2Dynamic, + kMxfp6E3M2Static, + kNvfp4Dynamic, + kNvfp4Static, ) from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api @@ -62,6 +103,576 @@ AITER_AVAILABLE = is_aiter_found_and_supported() DEVICE_TYPE = current_platform.device_type + +@dataclass(frozen=True) +class QTensorConfig: + name: str + weight: QuarkQTensorHint + input_tensors: QuarkQTensorHint + weight_quant_key: QuantKey | None = None + act_quant_key: QuantKey | None = None + dispatch_cls: type[QuarkScheme] | type[QuarkMoEMethod] | None = None + expected_error: tuple[type[Exception], str] | None = None + + +QTENSOR_CONFIGS = [ + QTensorConfig( + name="fp8_w8a8_static_tensor", + weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + }, + weight_quant_key=kFp8StaticTensorSym, + act_quant_key=kFp8StaticTensorSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_static_tensor_single_entry_lists", + weight=[ + { + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + } + ], + input_tensors=[ + { + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + } + ], + weight_quant_key=kFp8StaticTensorSym, + act_quant_key=kFp8StaticTensorSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_dynamic_tensor", + weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": True, + }, + weight_quant_key=kFp8StaticTensorSym, + act_quant_key=kFp8DynamicTensorSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_dynamic_token", + weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False}, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_channel", + "is_dynamic": True, + }, + weight_quant_key=kFp8StaticChannelSym, + act_quant_key=kFp8DynamicTokenSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_channel_static_tensor", + weight={"dtype": "fp8_e4m3", "qscheme": "per_channel", "is_dynamic": False}, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + }, + weight_quant_key=kFp8StaticChannelSym, + act_quant_key=kFp8StaticTensorSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_tensor_dynamic_token", + weight={"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_channel", + "is_dynamic": True, + }, + weight_quant_key=kFp8StaticTensorSym, + act_quant_key=kFp8DynamicTokenSym, + dispatch_cls=QuarkW8A8Fp8, + ), + QTensorConfig( + name="fp8_w8a8_dynamic_block_fp32", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + }, + weight_quant_key=kFp8Static128BlockSym, + act_quant_key=kFp8Dynamic128Sym, + dispatch_cls=QuarkW8A8Fp8PerBlock, + ), + QTensorConfig( + name="fp8_w8a8_dynamic_block_e8m0", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + "scale_type": "float8_e8m0fnu", + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + }, + weight_quant_key=kFp8Static128BlockE8M0Sym, + act_quant_key=kFp8Dynamic128Sym, + dispatch_cls=QuarkW8A8Fp8PerBlock, + ), + QTensorConfig( + name="fp8_w8a8_block_static_input", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": False, + "group_size": 128, + "symmetric": True, + }, + expected_error=(NotImplementedError, "No quark compatible scheme"), + ), + QTensorConfig( + name="fp8_w8a8_block_group_size_mismatch", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 64, + "symmetric": True, + }, + expected_error=(NotImplementedError, "No quark compatible scheme"), + ), + QTensorConfig( + name="fp8_w8a8_block_missing_block_size", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + }, + expected_error=(ValueError, "requires `block_size`"), + ), + QTensorConfig( + name="int8_w8a8_static_symmetric", + weight={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + weight_quant_key=kInt8StaticTensorSym, + act_quant_key=kInt8StaticTensorSym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_static_asymmetric", + weight={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": False, + }, + weight_quant_key=kInt8StaticTensorSym, + act_quant_key=kInt8StaticTensorAsym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_channel_static_symmetric", + weight={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + weight_quant_key=kInt8StaticChannelSym, + act_quant_key=kInt8StaticTensorSym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_channel_static_asymmetric", + weight={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": False, + }, + weight_quant_key=kInt8StaticChannelSym, + act_quant_key=kInt8StaticTensorAsym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_dynamic_tensor_symmetric", + weight={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": True, + "symmetric": True, + }, + weight_quant_key=kInt8StaticTensorSym, + act_quant_key=kInt8DynamicTensorSym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_dynamic_tensor_asymmetric", + weight={ + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": True, + "symmetric": False, + }, + weight_quant_key=kInt8StaticTensorSym, + act_quant_key=kInt8DynamicTensorAsym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_dynamic_token", + weight={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": True, + "symmetric": True, + }, + weight_quant_key=kInt8StaticChannelSym, + act_quant_key=kInt8DynamicTokenSym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="int8_w8a8_dynamic_token_asymmetric", + weight={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + }, + input_tensors={ + "dtype": "int8", + "qscheme": "per_channel", + "is_dynamic": True, + "symmetric": False, + }, + weight_quant_key=kInt8StaticChannelSym, + act_quant_key=kInt8DynamicTokenAsym, + dispatch_cls=QuarkW8A8Int8, + ), + QTensorConfig( + name="ocp_mx_mxfp4_weight_only", + weight={ + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors=None, + weight_quant_key=kMxfp4Static, + act_quant_key=None, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="ocp_mx_mxfp4_activation", + weight={ + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors={ + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + weight_quant_key=kMxfp4Static, + act_quant_key=kMxfp4Dynamic, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="ocp_mx_mxfp6_e3m2", + weight={ + "dtype": "fp6_e3m2", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors={ + "dtype": "fp6_e3m2", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + weight_quant_key=kMxfp6E3M2Static, + act_quant_key=kMxfp6E3M2Dynamic, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="ocp_mx_mxfp4_mxfp6_e3m2_activation", + weight={ + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors={ + "dtype": "fp6_e3m2", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + weight_quant_key=kMxfp4Static, + act_quant_key=kMxfp6E3M2Dynamic, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="ocp_mx_mxfp4_mxfp6_e2m3_activation", + weight={ + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors={ + "dtype": "fp6_e2m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + weight_quant_key=kMxfp4Static, + act_quant_key=kMxfp6E2M3Dynamic, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="ocp_mx_mxfp6_e2m3", + weight={ + "dtype": "fp6_e2m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + input_tensors={ + "dtype": "fp6_e2m3", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": True, + }, + weight_quant_key=kMxfp6E2M3Static, + act_quant_key=kMxfp6E2M3Dynamic, + dispatch_cls=QuarkOCP_MX, + ), + QTensorConfig( + name="nvfp4", + weight=[ + { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 16, + "is_dynamic": False, + }, + {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + ], + input_tensors=[ + { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 16, + "is_dynamic": True, + }, + {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + ], + weight_quant_key=kNvfp4Static, + act_quant_key=kNvfp4Dynamic, + dispatch_cls=QuarkNVFP4, + ), + QTensorConfig( + name="w4a8_fp8_static", + weight=[ + {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + { + "dtype": "int4", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + "ch_axis": 0, + }, + ], + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + }, + weight_quant_key=kInt4W4A8StaticChannelSym, + act_quant_key=kFp8StaticTensorSym, + dispatch_cls=QuarkW4A8Fp8MoEMethod, + ), + QTensorConfig( + name="w4a8_fp8_dynamic", + weight=[ + {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + { + "dtype": "int4", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + "ch_axis": 0, + }, + ], + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_channel", + "is_dynamic": True, + }, + weight_quant_key=kInt4W4A8StaticChannelSym, + act_quant_key=kFp8DynamicTokenSym, + dispatch_cls=QuarkW4A8Fp8MoEMethod, + ), + QTensorConfig( + name="w4a8_fp8_static_single_entry_input", + weight=[ + {"dtype": "fp8_e4m3", "qscheme": "per_tensor", "is_dynamic": False}, + { + "dtype": "int4", + "qscheme": "per_channel", + "is_dynamic": False, + "symmetric": True, + "ch_axis": 0, + }, + ], + input_tensors=[ + { + "dtype": "fp8_e4m3", + "qscheme": "per_tensor", + "is_dynamic": False, + } + ], + weight_quant_key=kInt4W4A8StaticChannelSym, + act_quant_key=kFp8StaticTensorSym, + dispatch_cls=QuarkW4A8Fp8MoEMethod, + ), +] + + +def _make_qtensor_config( + weight: QuarkQTensorHint, + input_tensors: QuarkQTensorHint, + exclude: list[str] | None = None, +) -> QuarkConfig: + return QuarkConfig( + { + "global_quant_config": { + "weight": weight, + "input_tensors": input_tensors, + }, + "layer_type_quant_config": {}, + "exclude": exclude or [], + } + ) + + +def _make_test_moe_config() -> FusedMoEConfig: + return FusedMoEConfig( + num_experts=8, + experts_per_token=2, + hidden_dim=256, + intermediate_size=256, + num_local_experts=8, + num_logical_experts=8, + activation=MoEActivation.SILU, + device=current_platform.device_type, + routing_method=RoutingMethodType.Renormalize, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + in_dtype=torch.bfloat16, + ) + + if QUARK_MXFP4_AVAILABLE: from quark.torch.export.nn.modules.realquantizer import StaticScaledRealQuantizer from quark.torch.kernel import mx as mx_kernel @@ -98,83 +709,197 @@ def test_quark_config_preserves_existing_packed_modules_mapping(): assert config.packed_modules_mapping["custom_proj"] == ["a", "b"] -def test_quark_fp8_w8a8_detects_per_block_config(): - config = QuarkConfig({}) - weight_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_block", - "is_dynamic": False, - "block_size": [128, 128], - "symmetric": True, - } - input_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_group", - "is_dynamic": True, - "group_size": 128, - "symmetric": True, - } +def test_quant_method_dispatch_ignored(default_vllm_config): + config = _make_qtensor_config(None, None, exclude=["linear", "experts"]) - assert config._is_fp8_w8a8(weight_config, input_config) + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + class TestRoutedExperts(RoutedExperts): + def __init__(self): + torch.nn.Module.__init__(self) + self.moe_config = _make_test_moe_config() + + assert config.get_quant_method_target("linear", LinearBase) == ( + None, + None, + UnquantizedLinearMethod, + ) + assert isinstance( + config.get_quant_method(TestLinear(), "linear"), UnquantizedLinearMethod + ) + + assert config.get_quant_method_target("experts", RoutedExperts) == ( + None, + None, + UnquantizedFusedMoEMethod, + ) + assert isinstance( + config.get_quant_method(TestRoutedExperts(), "experts"), + UnquantizedFusedMoEMethod, + ) + + dynamic_mxfp4_config = _make_qtensor_config( + { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "scale_format": "e8m0", + "is_dynamic": False, + }, + None, + exclude=["self_attn.q_proj", "mlp.down_proj"], + ) + dynamic_mxfp4_config.dynamic_mxfp4_quant = True + + assert dynamic_mxfp4_config.get_quant_method_target( + "self_attn.q_proj", LinearBase + ) == (kMxfp4Static, None, QuarkLinearMethod) + attention_proj = TestLinear() + assert isinstance( + dynamic_mxfp4_config.get_quant_method(attention_proj, "self_attn.q_proj"), + QuarkLinearMethod, + ) + assert isinstance(attention_proj.scheme, QuarkOCP_MX) + assert attention_proj.scheme.dynamic_mxfp4_quant + + assert dynamic_mxfp4_config.get_quant_method_target( + "mlp.down_proj", LinearBase + ) == (None, None, UnquantizedLinearMethod) + assert isinstance( + dynamic_mxfp4_config.get_quant_method(TestLinear(), "mlp.down_proj"), + UnquantizedLinearMethod, + ) -def test_quark_fp8_w8a8_rejects_per_block_static_input(): - config = QuarkConfig({}) - weight_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_block", - "is_dynamic": False, - "block_size": [128, 128], - "symmetric": True, - } - input_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_group", - "is_dynamic": False, - "group_size": 128, - "symmetric": True, - } +@pytest.mark.parametrize("case", QTENSOR_CONFIGS, ids=lambda case: case.name) +def test_quant_method_dispatch_target(case): + config = _make_qtensor_config(case.weight, case.input_tensors) + if case.expected_error is not None: + error_type, error_message = case.expected_error + with pytest.raises(error_type, match=error_message): + config.get_quant_method_target("linear", LinearBase) + return - assert not config._is_fp8_w8a8(weight_config, input_config) + assert case.dispatch_cls is not None + is_linear = issubclass(case.dispatch_cls, QuarkScheme) + + weight_quant_key, act_quant_key, method_cls = config.get_quant_method_target( + "linear" if is_linear else "experts", + LinearBase if is_linear else RoutedExperts, + ) + + assert weight_quant_key == case.weight_quant_key + assert act_quant_key == case.act_quant_key + assert method_cls is (QuarkLinearMethod if is_linear else case.dispatch_cls) -def test_quark_fp8_w8a8_rejects_per_block_group_size_mismatch(): - config = QuarkConfig({}) - weight_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_block", - "is_dynamic": False, - "block_size": [128, 128], - "symmetric": True, - } - input_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_group", - "is_dynamic": True, - "group_size": 64, - "symmetric": True, - } +@pytest.mark.parametrize( + ("weight", "input_tensors"), + [ + pytest.param( + { + "dtype": "int8", + "qscheme": "per_group", + "is_dynamic": False, + "symmetric": True, + }, + { + "dtype": "int8", + "qscheme": "per_tensor", + "is_dynamic": False, + "symmetric": True, + }, + id="single_entry", + ), + pytest.param( + [ + {"dtype": "int8", "qscheme": "per_tensor"}, + {"dtype": "int8", "qscheme": "per_tensor"}, + ], + [ + {"dtype": "int8", "qscheme": "per_tensor"}, + {"dtype": "int8", "qscheme": "per_tensor"}, + ], + id="multi_entry", + ), + ], +) +def test_quant_method_dispatch_unsupported(weight, input_tensors): + config = _make_qtensor_config(weight, input_tensors) - assert not config._is_fp8_w8a8(weight_config, input_config) + class TestRoutedExperts(RoutedExperts): + def __init__(self): + torch.nn.Module.__init__(self) + + with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"): + config.get_quant_method_target("experts", RoutedExperts) + + with pytest.raises(RuntimeError, match="^Unsupported FusedMoe scheme$"): + config.get_quant_method(TestRoutedExperts(), "experts") -def test_quark_w8a8_fp8_per_block_requires_block_size(): - weight_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_block", - "is_dynamic": False, - "symmetric": True, - } - input_config = { - "dtype": "fp8_e4m3", - "qscheme": "per_group", - "is_dynamic": True, - "group_size": 128, - "symmetric": True, - } +@pytest.mark.parametrize( + "case", + [case for case in QTENSOR_CONFIGS if case.expected_error is None], + ids=lambda case: case.name, +) +def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_config): + config = _make_qtensor_config(case.weight, case.input_tensors) + assert case.dispatch_cls is not None + if issubclass(case.dispatch_cls, QuarkScheme): - with pytest.raises(ValueError, match="requires `block_size`"): - QuarkW8A8Fp8PerBlock(weight_config, input_config) + class TestLinear(LinearBase): + def __init__(self): + torch.nn.Module.__init__(self) + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.quark.schemes." + "quark_w8a8_fp8.get_current_vllm_config", + lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)), + ) + layer = TestLinear() + method = config.get_quant_method(layer, "linear") + + assert isinstance(method, QuarkLinearMethod) + assert isinstance(layer.scheme, case.dispatch_cls) + if case.weight_quant_key == kFp8Static128BlockE8M0Sym: + # TODO: Remove once E8M0 quant key is properly handled in oracle + assert layer.scheme.weight_quant_key == kFp8Static128BlockSym + else: + assert layer.scheme.weight_quant_key == case.weight_quant_key + assert layer.scheme.activation_quant_key == case.act_quant_key + else: + + class TestRoutedExperts(RoutedExperts): + def __init__(self): + torch.nn.Module.__init__(self) + self.moe_config = _make_test_moe_config() + + for target in ( + "select_fp8_moe_backend", + "select_int8_moe_backend", + "select_mxfp4_moe_backend", + "backend_to_kernel_cls", + "select_nvfp4_moe_backend", + ): + monkeypatch.setattr( + f"vllm.model_executor.layers.quantization.quark.quark_moe.{target}", + lambda *args, **kwargs: (object(), object()), + ) + + # AssertionError: W4A8 FP8 MoE requires ROCm AITER fused MoE support + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.quark.quark_moe." + "rocm_aiter_ops.is_fused_moe_enabled", + lambda: True, + ) + + layer = TestRoutedExperts() + method = config.get_quant_method(layer, "experts") + + assert isinstance(method, case.dispatch_cls) @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 6c877b1a63..855a413c45 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import fnmatch -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NamedTuple, cast import torch from transformers import PretrainedConfig @@ -37,9 +37,35 @@ from vllm.model_executor.layers.quantization.quark.schemes import ( QuarkW8A8Int8, ) from vllm.model_executor.layers.quantization.quark.utils import ( + QuarkQTensorHint, deep_compare, should_ignore_layer, ) +from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( + _ACTIVATION_QUANT_KEY_MAP, + _WEIGHT_QUANT_KEY_MAP, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic128Sym, + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8Static128BlockE8M0Sym, + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kFp8StaticTensorSym, + kInt4W4A8StaticChannelSym, + kInt8DynamicTensorAsym, + kInt8DynamicTensorSym, + kInt8DynamicTokenAsym, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, + kInt8StaticTensorAsym, + kInt8StaticTensorSym, + kMxfp4Static, + kNvfp4Dynamic, + kNvfp4Static, +) from vllm.model_executor.models.utils import WeightsMapper from vllm.platforms import current_platform @@ -55,6 +81,15 @@ logger = init_logger(__name__) _DEEPSEEK_V3_FAMILY_MODEL_TYPES = frozenset({"deepseek_v3", "deepseek_v32"}) +class QuantKeyMatch(NamedTuple): + matches: bool + activation_quant_key: QuantKey | None + weight_quant_key: QuantKey | None + + def __bool__(self) -> bool: + return self.matches + + class QuarkConfig(QuantizationConfig): def __init__( self, @@ -155,41 +190,102 @@ class QuarkConfig(QuantizationConfig): def get_quant_method( self, layer: torch.nn.Module, prefix: str ) -> "QuantizeMethodBase | None": - # Check if the layer is skipped for quantization. + weight_quant_key, activation_quant_key, method_cls = ( + self.get_quant_method_target(prefix, type(layer)) + ) + exclude_layers = cast(list[str], self.quant_config.get("exclude")) - if should_ignore_layer( + is_ignored = should_ignore_layer( prefix, ignore=exclude_layers, fused_mapping=self.packed_modules_mapping, check_children=isinstance(layer, RoutedExperts), - ): - if isinstance(layer, RoutedExperts): - return UnquantizedFusedMoEMethod(layer.moe_config) - if ( - "self_attn" not in prefix # only quantize attention projections - or not getattr(self, "dynamic_mxfp4_quant", False) - or not isinstance(layer, LinearBase) # Ignore other methods - ): - return UnquantizedLinearMethod() + ) + dynamic_mxfp4_quant = ( + is_ignored and "self_attn" in prefix and self.dynamic_mxfp4_quant + ) - scheme = self.get_scheme( - layer=layer, - layer_name=prefix, - dynamic_mxfp4_quant=True, + if method_cls is UnquantizedFusedMoEMethod: + return UnquantizedFusedMoEMethod(layer.moe_config) + if method_cls is UnquantizedLinearMethod: + return UnquantizedLinearMethod() + if method_cls is QuarkLinearMethod: + _, _, scheme_cls = self.get_scheme_cls(type(layer), prefix) + scheme = self.init_scheme( + scheme_cls, + weight_quant_key=weight_quant_key, + activation_quant_key=activation_quant_key, + dynamic_mxfp4_quant=dynamic_mxfp4_quant, ) layer.scheme = scheme return QuarkLinearMethod(self) - if isinstance(layer, LinearBase): - scheme = self.get_scheme(layer=layer, layer_name=prefix) - layer.scheme = scheme - return QuarkLinearMethod(self) - if isinstance(layer, Attention): + if method_cls is QuarkKVCacheMethod: return QuarkKVCacheMethod(self) + if method_cls is not None and issubclass(method_cls, QuarkMoEMethod): + return QuarkMoEMethod.get_moe_method( + self, + module=layer, + method_cls=method_cls, + weight_quant_key=weight_quant_key, + activation_quant_key=activation_quant_key, + ) - if isinstance(layer, RoutedExperts): - return QuarkMoEMethod.get_moe_method(self, module=layer, layer_name=prefix) return None + def get_quant_method_target( + self, prefix: str, layer_type: type[torch.nn.Module] + ) -> tuple[ + QuantKey | None, + QuantKey | None, + type[QuantizeMethodBase] + | type[UnquantizedLinearMethod] + | type[UnquantizedFusedMoEMethod] + | None, + ]: + """Return weight key, activation key, and quant method class for + the given ``prefix`` and ``layer_type``. + + This is the counterpart of ``get_quant_method`` without quant method + instantiation. + + TODO: integrate as part of base QuantizationConfig.get_quant_method_target + in the future. + + ``None`` denotes an unquantized/BF16 activation. + """ + is_routed_experts = issubclass(layer_type, RoutedExperts) + if should_ignore_layer( + prefix, + ignore=cast(list[str], self.quant_config.get("exclude")), + fused_mapping=self.packed_modules_mapping, + check_children=is_routed_experts, + ): + if is_routed_experts: + return None, None, UnquantizedFusedMoEMethod + if issubclass(layer_type, LinearBase): + if "self_attn" in prefix and self.dynamic_mxfp4_quant: + weight_quant_key, activation_key, _ = self.get_scheme_cls( + layer_type, prefix + ) + return weight_quant_key, activation_key, QuarkLinearMethod + return None, None, UnquantizedLinearMethod + return None, None, None + if issubclass(layer_type, LinearBase): + weight_quant_key, activation_key, _ = self.get_scheme_cls( + layer_type, prefix + ) + return weight_quant_key, activation_key, QuarkLinearMethod + if issubclass(layer_type, Attention): + return None, None, QuarkKVCacheMethod + if is_routed_experts: + weight_quant_key, activation_quant_key, quant_method_cls = ( + QuarkMoEMethod.get_moe_method_target(self, layer_type, prefix) + ) + if quant_method_cls is None: + return None, None, None + return weight_quant_key, activation_quant_key, quant_method_cls + return None, None, None + @classmethod def from_config(cls, config: dict[str, Any]) -> "QuarkConfig": export_config = config.get("export") @@ -290,15 +386,20 @@ class QuarkConfig(QuantizationConfig): def _is_fp8_w4a8( self, - weight_quant: list[dict[str, Any]] | None, - input_quant: dict[str, Any] | None, - ) -> bool: + weight_quant: QuarkQTensorHint, + input_quant: QuarkQTensorHint, + ) -> QuantKeyMatch: # Confirm weights and input quantized. - if weight_quant is None or input_quant is None: - return False - - if not isinstance(weight_quant, list) or len(weight_quant) != 2: - return False + if isinstance(input_quant, list): + if len(input_quant) != 1: + return QuantKeyMatch(False, None, None) + input_quant = input_quant[0] + if ( + not isinstance(weight_quant, list) + or len(weight_quant) != 2 + or not isinstance(input_quant, dict) + ): + return QuantKeyMatch(False, None, None) # Confirm weight scheme is supported is_w4a8_dtype = ( @@ -321,24 +422,26 @@ class QuarkConfig(QuantizationConfig): and is_static_weight and is_per_tensor_fp8_and_per_channel_int4_weight ): - return False + return QuantKeyMatch(False, None, None) # Dynamic quantization is always supported if weights supported. if input_quant.get("is_dynamic"): - return True + return QuantKeyMatch(True, kFp8DynamicTokenSym, kInt4W4A8StaticChannelSym) # Confirm activation scheme is supported. is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor" - return is_per_tensor_activation + if not is_per_tensor_activation: + return QuantKeyMatch(False, None, None) + return QuantKeyMatch(True, kFp8StaticTensorSym, kInt4W4A8StaticChannelSym) def _is_fp8_w8a8( self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None, - ) -> bool: + ) -> QuantKeyMatch: # Confirm weights and input quantized. - if weight_quant is None or input_quant is None: - return False + if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict): + return QuantKeyMatch(False, None, None) # Confirm weight scheme is supported is_fp8_dtype = ( @@ -358,64 +461,125 @@ class QuarkConfig(QuantizationConfig): ) if not is_fp8_dtype or not is_static_weight: - return False + return QuantKeyMatch(False, None, None) if is_per_block_weight: if not input_quant.get("is_dynamic"): - return False - return ( + return QuantKeyMatch(False, None, None) + matches = ( input_quant.get("qscheme") == "per_group" and input_quant.get("group_size") == block_size[1] and input_quant.get("symmetric") is True ) + if not matches: + return QuantKeyMatch(False, None, None) + weight_quant_key = ( + kFp8Static128BlockE8M0Sym + if weight_quant.get("scale_type") == "float8_e8m0fnu" + else kFp8Static128BlockSym + ) + return QuantKeyMatch(True, kFp8Dynamic128Sym, weight_quant_key) if not is_per_tensor_or_channel_weight: - return False + return QuantKeyMatch(False, None, None) # Dynamic quantization is always supported if tensor/channel weights # are supported. if input_quant.get("is_dynamic"): - return True + activation_quant_key = ( + kFp8DynamicTokenSym + if input_quant.get("qscheme") == "per_channel" + else kFp8DynamicTensorSym + ) + weight_quant_key = ( + kFp8StaticChannelSym + if weight_quant.get("qscheme") == "per_channel" + else kFp8StaticTensorSym + ) + return QuantKeyMatch(True, activation_quant_key, weight_quant_key) # Confirm activation scheme is supported. is_per_tensor_activation = input_quant.get("qscheme") == "per_tensor" - return is_per_tensor_activation + if not is_per_tensor_activation: + return QuantKeyMatch(False, None, None) + weight_quant_key = ( + kFp8StaticChannelSym + if weight_quant.get("qscheme") == "per_channel" + else kFp8StaticTensorSym + ) + return QuantKeyMatch(True, kFp8StaticTensorSym, weight_quant_key) - def _is_static_tensor_w8a8( + def _is_w8a8_int8( self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None, - ) -> bool: + ) -> QuantKeyMatch: # Confirm weights and input quantized. - if weight_quant is None or input_quant is None: - return False + if not isinstance(weight_quant, dict) or not isinstance(input_quant, dict): + return QuantKeyMatch(False, None, None) is_int8_dtype = ( weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8" ) - is_tensor = ( - weight_quant.get("qscheme") in ["per_tensor", "per_channel"] - and input_quant.get("qscheme") == "per_tensor" + is_valid_weight_scheme = weight_quant.get("qscheme") in [ + "per_tensor", + "per_channel", + ] + is_static_input = input_quant.get( + "qscheme" + ) == "per_tensor" and not input_quant.get("is_dynamic") + is_dynamic_input = ( + input_quant.get("qscheme") == "per_channel" + and input_quant.get("is_dynamic") is True ) - - is_static = not weight_quant.get("is_dynamic") and not input_quant.get( - "is_dynamic" - ) - is_weight_symmetric = weight_quant.get("symmetric") is True # Both symmetric and asymmetric input quantization supported. # Only symmetric weight quantization supported. - return is_int8_dtype and is_tensor and is_weight_symmetric and is_static + matches = ( + is_int8_dtype + and is_valid_weight_scheme + and is_weight_symmetric + and ( + (is_static_input and not weight_quant.get("is_dynamic")) + or is_dynamic_input + ) + ) + if not matches: + return QuantKeyMatch(False, None, None) + weight_quant_key = ( + kInt8StaticChannelSym + if weight_quant.get("qscheme") == "per_channel" + else kInt8StaticTensorSym + ) + if is_static_input: + activation_quant_key = ( + kInt8StaticTensorSym + if input_quant.get("symmetric") is True + else kInt8StaticTensorAsym + ) + elif weight_quant.get("qscheme") == "per_channel": + activation_quant_key = ( + kInt8DynamicTokenSym + if input_quant.get("symmetric") is True + else kInt8DynamicTokenAsym + ) + else: + activation_quant_key = ( + kInt8DynamicTensorSym + if input_quant.get("symmetric") is True + else kInt8DynamicTensorAsym + ) + return QuantKeyMatch(True, activation_quant_key, weight_quant_key) def _is_w4a8_mxfp4_fp8( self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None, - ) -> bool: + ) -> QuantKeyMatch: if weight_quant is None or input_quant is None: - return False + return QuantKeyMatch(False, None, None) is_weight_mxfp4 = ( weight_quant.get("dtype") == "fp4" @@ -432,53 +596,24 @@ class QuarkConfig(QuantizationConfig): and input_quant.get("symmetric") is True # Symmetric quantization ) - return is_weight_mxfp4 and is_input_fp8 - - def _is_dynamic_per_token_w8a8( - self, - weight_quant: dict[str, Any] | None, - input_quant: dict[str, Any] | None, - ) -> bool: - """Detect W8A8 INT8 with per-tensor or per-channel - weights and dynamic per-token input.""" - if weight_quant is None or input_quant is None: - return False - - is_int8_dtype = ( - weight_quant.get("dtype") == "int8" and input_quant.get("dtype") == "int8" - ) - - is_valid_weight_scheme = weight_quant.get("qscheme") in [ - "per_tensor", - "per_channel", - ] - is_per_token_input = input_quant.get("qscheme") == "per_channel" - - is_dynamic_input = input_quant.get("is_dynamic") is True - is_weight_symmetric = weight_quant.get("symmetric") is True - - return ( - is_int8_dtype - and is_valid_weight_scheme - and is_per_token_input - and is_dynamic_input - and is_weight_symmetric - ) + if not (is_weight_mxfp4 and is_input_fp8): + return QuantKeyMatch(False, None, None) + return QuantKeyMatch(True, kFp8StaticTensorSym, kMxfp4Static) def _is_nvfp4( self, - weight_quant: dict[str, Any] | list[dict[str, Any]] | None, - input_quant: dict[str, Any] | list[dict[str, Any]] | None, - ) -> bool: + weight_quant: QuarkQTensorHint, + input_quant: QuarkQTensorHint, + ) -> QuantKeyMatch: # Confirm weights and input quantized. if weight_quant is None or input_quant is None: - return False + return QuantKeyMatch(False, None, None) # Confirm both weight_quant and input_quant are lists with 2 elements if not isinstance(weight_quant, list) or len(weight_quant) != 2: - return False + return QuantKeyMatch(False, None, None) if not isinstance(input_quant, list) or len(input_quant) != 2: - return False + return QuantKeyMatch(False, None, None) # First element should be fp4 with per_group quantization is_fp4_per_group_weight = ( @@ -506,36 +641,35 @@ class QuarkConfig(QuantizationConfig): and not input_quant[1].get("is_dynamic") ) - return ( + matches = ( is_fp4_per_group_weight # type: ignore[return-value] and is_fp4_per_group_input and is_fp8_per_tensor_weight and is_fp8_per_tensor_input ) + if not matches: + return QuantKeyMatch(False, None, None) + return QuantKeyMatch(True, kNvfp4Dynamic, kNvfp4Static) def _is_w_ocp_mx_a_x( - self, weight_quant: dict[str, Any] | None, input_quant: dict[str, Any] | None - ) -> bool: + self, + weight_quant: dict[str, Any] | None, + input_quant: dict[str, Any] | None, + allow_static_fp8: bool = False, + ) -> QuantKeyMatch: """ This check returns True only if it is an OCP-MX weight quantization. - The activation can be any data type (e.g., FP16/BF16, FP8, or OCP-MX format). + The activation can be FP16/BF16, OCP MXFP4, MXFP8, FP8. The rationale for checking only the weight type is that the model loading concept and process primarily concerns the weights themselves. """ # Confirm weights quantized. - if weight_quant is None: + if not isinstance(weight_quant, dict): logger.debug( "Quark model's weight quantization is incompatible with OCP_MX format: " - "weight_quant is not set." + "weight_quant is not a dictionary." ) - return False - - if isinstance(weight_quant, list): - logger.debug( - "Quark model's weight quantization is incompatible with OCP_MX format: " - "weight_quant is a list (e.g. fp8_w4a8), OCP_MX requires a single dict." - ) - return False + return QuantKeyMatch(False, None, None) # Input and weight qscheme needs to be per group. if weight_quant.get("qscheme") != "per_group": @@ -543,7 +677,7 @@ class QuarkConfig(QuantizationConfig): "Quark model's weight quantization is incompatible with OCP MX format: " "weight is not per_group." ) - return False + return QuantKeyMatch(False, None, None) # Input and weight group size needs to be 32. if weight_quant.get("group_size") != 32: @@ -551,7 +685,7 @@ class QuarkConfig(QuantizationConfig): "Quark model's weight quantization is incompatible with OCP MX format: " "group_size of weight is not 32." ) - return False + return QuantKeyMatch(False, None, None) # Activations and weight scales need to be in e8m0 format. if weight_quant.get("scale_format") != "e8m0": @@ -559,7 +693,7 @@ class QuarkConfig(QuantizationConfig): "Quark model's weight quantization is incompatible with OCP MX format: " "scale_format of weight is not e8m0." ) - return False + return QuantKeyMatch(False, None, None) # Input and weight dtypes need to be any of fp4, # fp6_e3m2 or fp6_e3m2, possibly mixed. @@ -572,9 +706,48 @@ class QuarkConfig(QuantizationConfig): "Quark model's weight quantization is incompatible with OCP MX format: " "dtype is not in {fp4, fp6_e3m2, fp6_e2m3}." ) - return False + return QuantKeyMatch(False, None, None) - return True + weight_dtype = weight_quant["dtype"].replace("fp", "mxfp") + weight_quant_key = _WEIGHT_QUANT_KEY_MAP[weight_dtype] + if input_quant is None: + activation_quant_key = None + elif not input_quant.get("is_dynamic"): + if ( + allow_static_fp8 + and input_quant.get("dtype") == "fp8_e4m3" + and input_quant.get("qscheme") == "per_tensor" + and input_quant.get("symmetric") is True + ): + activation_quant_key = kFp8StaticTensorSym + else: + logger.debug( + "Quark model's OCP MX quantization is incompatible with static " + "input scales." + ) + return QuantKeyMatch(False, None, None) + elif input_quant["dtype"] == "fp8_e4m3": + activation_quant_key = kFp8DynamicTensorSym + else: + input_dtype = input_quant["dtype"].replace("fp", "mxfp") + if input_dtype not in _ACTIVATION_QUANT_KEY_MAP: + raise ValueError( + f"Unsupported input_dtype={input_dtype} in Quark's vLLM " + "integration. Supported activation dtypes are " + f"{_ACTIVATION_QUANT_KEY_MAP.keys()}, or None for " + "weight-only quantization." + ) + activation_quant_key = _ACTIVATION_QUANT_KEY_MAP[input_dtype] + return QuantKeyMatch(True, activation_quant_key, weight_quant_key) + + @staticmethod + def _unwrap_single_quant_config( + quant_config: QuarkQTensorHint, + ) -> dict[str, Any] | None: + """Unwrap one-entry quantization-config lists.""" + if isinstance(quant_config, list): + return quant_config[0] if len(quant_config) == 1 else None + return quant_config def get_layer_quant_config_from_name( self, layer_name: str @@ -617,14 +790,14 @@ class QuarkConfig(QuantizationConfig): return None def _find_matched_config( - self, layer_name: str, module: torch.nn.Module + self, layer_name: str, module: torch.nn.Module | type[torch.nn.Module] ) -> dict[str, Any]: # Priority order: # 1. layer_quant_config, # 2. layer_type_quant_config, # 3. global_quant_config. - layer_type = cast(str, type(module)) + layer_type = cast(str, module if isinstance(module, type) else type(module)) layer_type_quant_config = cast( dict[str, Any], self.quant_config.get("layer_type_quant_config") ) @@ -662,51 +835,71 @@ class QuarkConfig(QuantizationConfig): return layer_quant_config return fallback_config - def _get_scheme_from_config( - self, config: dict[str, Any], dynamic_mxfp4_quant: bool = False - ) -> "QuarkScheme": + def _get_scheme_cls_from_config( + self, config: dict[str, Any] + ) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]: if config.get("output_tensors") or config.get("bias"): raise NotImplementedError( "Currently, Quark models with output_tensors " "and bias quantized are not supported" ) - weight_config = cast(dict[str, Any], config.get("weight")) - input_config = cast(dict[str, Any], config.get("input_tensors")) + weight_config = cast(QuarkQTensorHint, config.get("weight")) + input_config = cast(QuarkQTensorHint, config.get("input_tensors")) - if self._is_nvfp4(weight_config, input_config): - return QuarkNVFP4() - elif self._is_fp8_w8a8(weight_config, input_config): - is_fp8_w8a8_supported = self._check_scheme_supported( - QuarkW8A8Fp8.get_min_capability(), error=False + if ( + isinstance(weight_config, list) + and len(weight_config) > 1 + or isinstance(input_config, list) + and len(input_config) > 1 + ): + if match := self._is_nvfp4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkNVFP4, + ) + + raise NotImplementedError( + "Multi-entry weight or activation quantization configs are only " + "supported for NVFP4." ) - if is_fp8_w8a8_supported: - if weight_config.get("qscheme") == "per_block": - return QuarkW8A8Fp8PerBlock(weight_config, input_config) - return QuarkW8A8Fp8(weight_config, input_config) - elif self._is_static_tensor_w8a8(weight_config, input_config): - weight_qscheme = cast(str, weight_config.get("qscheme")) - return QuarkW8A8Int8( - qscheme=weight_qscheme, - is_static_input_scheme=True, - input_symmetric=input_config.get("symmetric"), + + weight_config = self._unwrap_single_quant_config(weight_config) + input_config = self._unwrap_single_quant_config(input_config) + + if ( + isinstance(weight_config, dict) + and weight_config.get("qscheme") == "per_block" + and not weight_config.get("block_size") + ): + raise ValueError( + "Quark W8A8 FP8 per-block weight quantization requires " + "`block_size` in the weight quantization config." ) - elif self._is_w4a8_mxfp4_fp8(weight_config, input_config): - is_w4a8_supported = self._check_scheme_supported( - QuarkW4A8_MXFP4_FP8.get_min_capability(), error=False + + if match := self._is_fp8_w8a8(weight_config, input_config): + assert weight_config is not None + if weight_config.get("qscheme") == "per_block": + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW8A8Fp8PerBlock, + ) + return match.weight_quant_key, match.activation_quant_key, QuarkW8A8Fp8 + elif match := self._is_w8a8_int8(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW8A8Int8, ) - if is_w4a8_supported: - return QuarkW4A8_MXFP4_FP8(weight_config, input_config) - elif self._is_dynamic_per_token_w8a8(weight_config, input_config): - weight_qscheme = cast(str, weight_config.get("qscheme")) - return QuarkW8A8Int8( - qscheme=weight_qscheme, - is_static_input_scheme=False, - input_symmetric=input_config.get("symmetric"), - ) - elif self._is_w_ocp_mx_a_x(weight_config, input_config): - return QuarkOCP_MX( - weight_config, input_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant + elif match := self._is_w4a8_mxfp4_fp8(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A8_MXFP4_FP8, ) + elif match := self._is_w_ocp_mx_a_x(weight_config, input_config): + return match.weight_quant_key, match.activation_quant_key, QuarkOCP_MX raise NotImplementedError( "No quark compatible scheme was found. " @@ -714,19 +907,51 @@ class QuarkConfig(QuantizationConfig): f"Input config: {input_config}" ) - def get_scheme( - self, layer: torch.nn.Module, layer_name: str, dynamic_mxfp4_quant: bool = False - ) -> "QuarkScheme": - layer_quant_config = self._find_matched_config(layer_name, layer) - - # Find the quant_scheme - scheme = self._get_scheme_from_config( - layer_quant_config, dynamic_mxfp4_quant=dynamic_mxfp4_quant + def get_scheme_cls( + self, layer_type: type[torch.nn.Module], layer_name: str + ) -> tuple[QuantKey | None, QuantKey | None, type["QuarkScheme"]]: + """Return quantization keys and scheme class without initializing it.""" + return self._get_scheme_cls_from_config( + self._find_matched_config(layer_name, layer_type) ) + + def init_scheme( + self, + scheme_cls: type["QuarkScheme"], + weight_quant_key: QuantKey | None, + activation_quant_key: QuantKey | None, + dynamic_mxfp4_quant: bool = False, + ) -> "QuarkScheme": + """Construct a Quark scheme selected by get_scheme_cls.""" + if scheme_cls not in ( + QuarkW8A8Fp8, + QuarkW8A8Fp8PerBlock, + QuarkW8A8Int8, + QuarkOCP_MX, + QuarkNVFP4, + QuarkW4A8_MXFP4_FP8, + ): + raise AssertionError(f"Unsupported Quark scheme class: {scheme_cls}") + + kwargs: dict[str, Any] = {"activation_quant_key": activation_quant_key} + + # These classes handle different possible `weight_quant_key`. + if scheme_cls in ( + QuarkW8A8Fp8, + QuarkW8A8Fp8PerBlock, + QuarkW8A8Int8, + QuarkOCP_MX, + ): + kwargs["weight_quant_key"] = weight_quant_key + + if scheme_cls is QuarkOCP_MX: + kwargs["dynamic_mxfp4_quant"] = dynamic_mxfp4_quant + + scheme = scheme_cls(**kwargs) + # Raise error if device does not support the scheme # (e.g. fp8 needs ada lovelace) self._check_scheme_supported(scheme.get_min_capability()) - return scheme @staticmethod diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index d2b286f148..74a64041cb 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import Any +from typing import TYPE_CHECKING import torch @@ -56,19 +56,27 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( make_nvfp4_moe_quant_config, select_nvfp4_moe_backend, ) +from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( + _ACTIVATION_QUANT_KEY_MAP, + _WEIGHT_QUANT_KEY_MAP, OCP_MX_BLOCK_SIZE, OCP_MX_Scheme, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, + QuantKey, kFp8DynamicTensorSym, kFp8DynamicTokenSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4W4A8StaticChannelSym, + kInt8DynamicTensorAsym, kInt8DynamicTensorSym, + kInt8DynamicTokenAsym, kInt8DynamicTokenSym, kInt8StaticChannelSym, + kInt8StaticTensorAsym, kInt8StaticTensorSym, kMxfp4Dynamic, kNvfp4Dynamic, @@ -82,6 +90,9 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( from vllm.model_executor.utils import replace_parameter, set_weight_attrs from vllm.platforms import current_platform +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig + logger = init_logger(__name__) __all__ = [ @@ -93,67 +104,154 @@ __all__ = [ class QuarkMoEMethod(FusedMoEMethodBase): - def __init__(self, moe: FusedMoEConfig): + supported_activation_quant_keys: list[QuantKey | None] = [] + supported_weight_quant_keys: list[QuantKey] = [] + + def __init__( + self, + moe: FusedMoEConfig, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + ): super().__init__(moe) + if activation_quant_key not in self.supported_activation_quant_keys: + raise ValueError( + f"Unsupported activation quant key: {activation_quant_key}" + ) + if weight_quant_key not in self.supported_weight_quant_keys: + raise ValueError(f"Unsupported weight quant key: {weight_quant_key}") + self.activation_quant_key = activation_quant_key + self.weight_quant_key = weight_quant_key self.has_bias = self.moe.has_bias @staticmethod - def get_moe_method( - quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821 - module: RoutedExperts, + def get_moe_method_target( + quant_config: "QuarkConfig", + layer_type: type[torch.nn.Module], layer_name: str, - ) -> "QuarkMoEMethod": - layer_quant_config = quant_config._find_matched_config(layer_name, module) - + ) -> tuple[ + QuantKey | None, + QuantKey | None, + type["QuarkMoEMethod"] | None, + ]: + """Return the quantization target for a routed-experts layer.""" + layer_quant_config = quant_config._find_matched_config(layer_name, layer_type) if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"): raise NotImplementedError( "Currently, Quark models with " "output_tensors and bias " "quantized are not supported" ) + weight_config: QuarkQTensorHint = layer_quant_config.get("weight") + input_config: QuarkQTensorHint = layer_quant_config.get("input_tensors") - weight_config = layer_quant_config.get("weight") - input_config = layer_quant_config.get("input_tensors") + if ( + isinstance(weight_config, list) + and len(weight_config) > 1 + or isinstance(input_config, list) + and len(input_config) > 1 + ): + if match := quant_config._is_fp8_w4a8(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW4A8Fp8MoEMethod, + ) + if match := quant_config._is_nvfp4(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkNvfp4MoEMethod, + ) + raise RuntimeError("Unsupported FusedMoe scheme") - if quant_config._is_fp8_w4a8(weight_config, input_config): - return QuarkW4A8Fp8MoEMethod(weight_config, input_config, module.moe_config) - elif quant_config._is_nvfp4(weight_config, input_config): - return QuarkNvfp4MoEMethod( - weight_config, input_config, module.moe_config, quant_config + weight_config = quant_config._unwrap_single_quant_config(weight_config) + input_config = quant_config._unwrap_single_quant_config(input_config) + + if match := quant_config._is_fp8_w8a8(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW8A8Fp8MoEMethod, ) - elif quant_config._is_fp8_w8a8(weight_config, input_config): - return QuarkW8A8Fp8MoEMethod(weight_config, input_config, module.moe_config) - elif quant_config._is_w_ocp_mx_a_x(weight_config, input_config): + if match := quant_config._is_w_ocp_mx_a_x( + weight_config, input_config, allow_static_fp8=True + ): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkOCP_MX_MoEMethod, + ) + if match := quant_config._is_w8a8_int8(weight_config, input_config): + return ( + match.weight_quant_key, + match.activation_quant_key, + QuarkW8A8Int8MoEMethod, + ) + raise RuntimeError("Unsupported FusedMoe scheme") + + @staticmethod + def get_moe_method( + quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821 + module: RoutedExperts, + method_cls: type["QuarkMoEMethod"], + weight_quant_key: QuantKey | None, + activation_quant_key: QuantKey | None, + ) -> "QuarkMoEMethod": + if weight_quant_key is None: + raise RuntimeError("Unsupported FusedMoe scheme") + + if method_cls is QuarkW4A8Fp8MoEMethod: + return QuarkW4A8Fp8MoEMethod(module.moe_config, activation_quant_key) + elif method_cls is QuarkNvfp4MoEMethod: + return QuarkNvfp4MoEMethod( + module.moe_config, quant_config, activation_quant_key + ) + elif method_cls is QuarkW8A8Fp8MoEMethod: + return QuarkW8A8Fp8MoEMethod( + module.moe_config, weight_quant_key, activation_quant_key + ) + elif method_cls is QuarkOCP_MX_MoEMethod: # All OCP MX schemes (W4A16, W4A8, etc.) handled by QuarkOCP_MX_MoEMethod # Backend selection happens inside via oracle - return QuarkOCP_MX_MoEMethod(weight_config, input_config, module.moe_config) - elif quant_config._is_static_tensor_w8a8( - weight_config, input_config - ) or quant_config._is_dynamic_per_token_w8a8(weight_config, input_config): + return QuarkOCP_MX_MoEMethod( + module.moe_config, weight_quant_key, activation_quant_key + ) + elif method_cls is QuarkW8A8Int8MoEMethod: return QuarkW8A8Int8MoEMethod( - weight_config, input_config, module.moe_config + module.moe_config, weight_quant_key, activation_quant_key ) else: raise RuntimeError("Unsupported FusedMoe scheme") class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): + supported_activation_quant_keys = [ + kFp8StaticTensorSym, + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + ] + supported_weight_quant_keys = [ + kFp8StaticChannelSym, + kFp8StaticTensorSym, + ] + def __init__( self, - weight_config: dict[str, Any], - input_config: dict[str, Any], moe: FusedMoEConfig, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - super().__init__(moe) - self.weight_quant = weight_config - self.input_quant = input_config - - self.weight_qscheme = self.weight_quant.get("qscheme") - self.input_qscheme = self.input_quant.get("qscheme") - self.weight_dtype = self.weight_quant.get("dtype", "").replace( - "fp8_e4m3", "fp8" + super().__init__(moe, weight_quant_key, activation_quant_key) + self.weight_qscheme = ( + "per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor" ) - self.input_dtype = self.input_quant.get("dtype", "").replace("fp8_e4m3", "fp8") + self.input_qscheme = ( + "per_channel" + if activation_quant_key == kFp8DynamicTokenSym + else "per_tensor" + ) + self.static_input_scales = activation_quant_key == kFp8StaticTensorSym per_tensor = ( self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor" ) @@ -170,28 +268,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): f"{self.weight_qscheme}, {self.input_qscheme}" ) # noqa E501 - self.static_input_scales = not self.input_quant.get("is_dynamic") - if self.static_input_scales and per_channel: - raise ValueError( - "For FP8 Fused MoE layer, we require either per tensor or " - "channelwise, dynamic per token quantization." - ) - - # Determine quant keys for oracle backend selection - if per_channel: - weight_key = kFp8StaticChannelSym - activation_key = kFp8DynamicTokenSym - elif self.static_input_scales: - weight_key = kFp8StaticTensorSym - activation_key = kFp8StaticTensorSym - else: - weight_key = kFp8StaticTensorSym - activation_key = kFp8DynamicTensorSym - self.fp8_backend, self.experts_cls = select_fp8_moe_backend( config=moe, - weight_key=weight_key, - activation_key=activation_key, + weight_key=self.weight_quant_key, + activation_key=self.activation_quant_key, ) self.model_type = getattr( @@ -500,17 +580,41 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): """Quark W8A8 INT8 MoE method.""" + supported_activation_quant_keys = [ + kInt8StaticTensorSym, + kInt8StaticTensorAsym, + kInt8DynamicTensorSym, + kInt8DynamicTensorAsym, + kInt8DynamicTokenSym, + kInt8DynamicTokenAsym, + ] + supported_weight_quant_keys = [ + kInt8StaticChannelSym, + kInt8StaticTensorSym, + ] + def __init__( self, - weight_config: dict[str, Any], - input_config: dict[str, Any], moe: FusedMoEConfig, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - super().__init__(moe) - self.weight_quant = weight_config - self.input_quant = input_config - self.weight_qscheme = self.weight_quant.get("qscheme", "per_tensor") - self.static_input_scales = not self.input_quant.get("is_dynamic", False) + super().__init__(moe, weight_quant_key, activation_quant_key) + + # TODO: oracle needs to support asym int8. + if self.activation_quant_key == kInt8StaticTensorAsym: + self.activation_quant_key = kInt8StaticTensorSym + if self.activation_quant_key == kInt8DynamicTensorAsym: + self.activation_quant_key = kInt8DynamicTensorSym + if self.activation_quant_key == kInt8DynamicTokenAsym: + self.activation_quant_key = kInt8DynamicTokenSym + + self.weight_qscheme = ( + "per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor" + ) + + assert self.activation_quant_key is not None + self.static_input_scales = self.activation_quant_key.scale.static self.moe_quant_config: FusedMoEQuantConfig | None = None self.moe_kernel: mk.FusedMoEKernel | None = None @@ -528,17 +632,10 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): # Map the Quark weight scheme to oracle quant keys. Per-channel # weights pair with dynamic per-token activations; per-tensor # weights with dynamic per-tensor activations. - if self.weight_qscheme == "per_channel": - weight_key = kInt8StaticChannelSym - activation_key = kInt8DynamicTokenSym - else: - weight_key = kInt8StaticTensorSym - activation_key = kInt8DynamicTensorSym - self.int8_backend, self.experts_cls = select_int8_moe_backend( config=moe, - weight_key=weight_key, - activation_key=activation_key, + weight_key=self.weight_quant_key, + activation_key=self.activation_quant_key, ) def create_weights( @@ -876,15 +973,18 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): + supported_activation_quant_keys = [ + kFp8DynamicTokenSym, + kFp8StaticTensorSym, + ] + supported_weight_quant_keys = [kInt4W4A8StaticChannelSym] + def __init__( self, - weight_config: dict[str, Any], - input_config: dict[str, Any], moe: FusedMoEConfig, + activation_quant_key: QuantKey | None, ): - super().__init__(moe) - self.weight_quant = weight_config - self.input_quant = input_config + super().__init__(moe, kInt4W4A8StaticChannelSym, activation_quant_key) assert rocm_aiter_ops.is_fused_moe_enabled(), ( "W4A8 FP8 MoE requires ROCm AITER fused MoE support." @@ -1032,37 +1132,36 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): + supported_activation_quant_keys = [ + *_ACTIVATION_QUANT_KEY_MAP.values(), + kFp8DynamicTensorSym, + kFp8StaticTensorSym, + None, + ] + supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()] + def __init__( self, - weight_config: dict[str, Any], - input_config: dict[str, Any] | None, moe: FusedMoEConfig, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - super().__init__(moe) - self.weight_quant = weight_config - self.input_quant = input_config - - weight_qscheme = self.weight_quant.get("qscheme") - if not weight_qscheme == "per_group": - raise ValueError( - "For MX(FP4) Fused MoE layers, only per-group scales " - f"for weights are supported. Found {weight_qscheme}." - ) # noqa E501 - - self.weight_dtype = self.weight_quant["dtype"].replace("fp", "mxfp") - if self.input_quant is not None: - input_quant = self.input_quant["dtype"] - if input_quant in ["fp4", "fp6_e3m2", "fp6_e2m3"]: - self.input_dtype = input_quant.replace("fp", "mxfp") - elif input_quant == "fp8_e4m3": - self.input_dtype = input_quant.replace("fp8_e4m3", "fp8") - else: - raise NotImplementedError( - f"Current input dtype {input_quant} is not compatible \ - with OCP MX (weight) MoE quantization. Please open an issue" - ) - else: + super().__init__(moe, weight_quant_key, activation_quant_key) + self.weight_dtype = next( + dtype + for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() + if quant_key == weight_quant_key + ) + if activation_quant_key in {kFp8DynamicTensorSym, kFp8StaticTensorSym}: + self.input_dtype: str | None = "fp8" + elif activation_quant_key is None: self.input_dtype = None + else: + self.input_dtype = next( + dtype + for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items() + if quant_key == activation_quant_key + ) self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype( self.input_dtype, self.weight_dtype @@ -1085,10 +1184,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): self.w13_precision_config = None self.w2_precision_config = None - if self.input_quant is not None: - self.static_input_scales = not self.input_quant.get("is_dynamic") - else: - self.static_input_scales = False + self.static_input_scales = activation_quant_key == kFp8StaticTensorSym # Select backend based on OCP MX scheme if self.ocp_mx_scheme == "w_mxfp4": @@ -1388,6 +1484,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): f"in vLLM for OCP MX scheme {self.ocp_mx_scheme}. Please open an issue." ) else: + assert self.input_dtype is not None return ocp_mx_moe_quant_config( quant_dtype=self.input_dtype, weight_dtype=self.weight_dtype, @@ -1459,24 +1556,24 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): class QuarkNvfp4MoEMethod(QuarkMoEMethod): + supported_activation_quant_keys = [kNvfp4Dynamic] + supported_weight_quant_keys = [kNvfp4Static] + def __init__( self, - weight_config: dict[str, Any], - input_config: dict[str, Any], moe: FusedMoEConfig, quant_config: "QuarkConfig", # type: ignore # noqa E501 # noqa F821 + activation_quant_key: QuantKey | None, ): - super().__init__(moe) - self.weight_quant = weight_config - self.input_quant = input_config + super().__init__(moe, kNvfp4Static, activation_quant_key) self.quant_config = quant_config self.group_size = 16 # Select experts implementation. self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend( config=self.moe, - weight_key=kNvfp4Static, - activation_key=kNvfp4Dynamic, + weight_key=self.weight_quant_key, + activation_key=self.activation_quant_key, ) def create_weights( diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_nvfp4.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_nvfp4.py index d8a339770f..7fb0332b57 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_nvfp4.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_nvfp4.py @@ -14,6 +14,11 @@ from vllm.model_executor.kernels.linear.nvfp4.emulation import ( from vllm.model_executor.layers.quantization.quark.schemes.quark_scheme import ( QuarkScheme, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kNvfp4Dynamic, + kNvfp4Static, +) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, ModelWeightParameter, @@ -36,9 +41,14 @@ class QuarkNVFP4(QuarkScheme): - input_scale_2: bfloat16/float32, scalar (global input scale) """ + supported_activation_quant_keys = [kNvfp4Dynamic] + supported_weight_quant_keys = [kNvfp4Static] + def __init__( self, + activation_quant_key: QuantKey | None, ): + super().__init__(kNvfp4Static, activation_quant_key) self.kernel = init_nvfp4_linear_kernel() self.group_size = 16 diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py index cea1c55ac5..30758782c3 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_ocp_mx.py @@ -3,7 +3,6 @@ from collections.abc import Callable from fractions import Fraction -from typing import Any import torch @@ -15,15 +14,14 @@ from vllm.model_executor.kernels.linear import ( init_mxfp6_linear_kernel, ) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( + _ACTIVATION_QUANT_KEY_MAP, + _WEIGHT_QUANT_KEY_MAP, OCP_MX_BLOCK_SIZE, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, - kMxfp4Dynamic, kMxfp4Static, - kMxfp6E2M3Dynamic, kMxfp6E2M3Static, - kMxfp6E3M2Dynamic, kMxfp6E3M2Static, ) from vllm.model_executor.parameter import ( @@ -38,45 +36,32 @@ from .quark_scheme import QuarkScheme logger = init_logger(__name__) -_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = { - "mxfp4": kMxfp4Static, - "mxfp6_e3m2": kMxfp6E3M2Static, - "mxfp6_e2m3": kMxfp6E2M3Static, -} - -_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = { - "mxfp4": kMxfp4Dynamic, - "mxfp6_e3m2": kMxfp6E3M2Dynamic, - "mxfp6_e2m3": kMxfp6E2M3Dynamic, -} - class QuarkOCP_MX(QuarkScheme): ocp_mx_linear: MxFp6LinearKernel | MxFp4LinearKernel + supported_activation_quant_keys = [*_ACTIVATION_QUANT_KEY_MAP.values(), None] + supported_weight_quant_keys = [*_WEIGHT_QUANT_KEY_MAP.values()] def __init__( self, - weight_quant_spec: dict[str, Any], - input_quant_spec: dict[str, Any] | None, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, dynamic_mxfp4_quant: bool = False, ): + super().__init__(weight_quant_key, activation_quant_key) self.dynamic_mxfp4_quant = dynamic_mxfp4_quant - self.weight_dtype = weight_quant_spec["dtype"].replace("fp", "mxfp") - self.input_dtype: str | None = None - if input_quant_spec is not None: - self.input_dtype = input_quant_spec["dtype"].replace("fp", "mxfp") - - if self.input_dtype not in [None, *_ACTIVATION_QUANT_KEY_MAP]: - raise ValueError( - f"Unsupported input_dtype={self.input_dtype} for QuarkOCP_MX. " - f"Supported activation dtypes are {_ACTIVATION_QUANT_KEY_MAP.keys()}, " - "or None for weight-only quantization." + self.weight_dtype = next( + dtype + for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items() + if quant_key == weight_quant_key + ) + self.input_dtype = ( + next( + dtype + for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items() + if quant_key == activation_quant_key ) - - self.weight_quant_key = _WEIGHT_QUANT_KEY_MAP[self.weight_dtype] - self.activation_quant_key = ( - _ACTIVATION_QUANT_KEY_MAP[self.input_dtype] - if self.input_dtype is not None + if activation_quant_key is not None else None ) @@ -85,17 +70,6 @@ class QuarkOCP_MX(QuarkScheme): else: self.packed_factor = Fraction(numerator=8, denominator=6) - if input_quant_spec is None: - self.static_input_scales = False - else: - self.static_input_scales = not input_quant_spec.get("is_dynamic") - - if self.static_input_scales: - raise NotImplementedError( - "QuarkOCP_MX with static input scales is currently not " - "implemented. Please open an issue." - ) - if not current_platform.supports_mx(): logger.warning_once( "The current platform does not support native MXFP4/MXFP6 " diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py index 6f8db9ea57..419055b372 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py @@ -5,6 +5,8 @@ from abc import ABC, abstractmethod import torch +from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey + __all__ = ["QuarkScheme"] @@ -14,6 +16,23 @@ class QuarkScheme(ABC): of different quantization schemes supported by Quark. """ + supported_activation_quant_keys: list[QuantKey | None] = [] + supported_weight_quant_keys: list[QuantKey] = [] + + def __init__( + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, + ): + if activation_quant_key not in self.supported_activation_quant_keys: + raise ValueError( + f"Unsupported activation quant key: {activation_quant_key}" + ) + if weight_quant_key not in self.supported_weight_quant_keys: + raise ValueError(f"Unsupported weight quant key: {weight_quant_key}") + self.activation_quant_key = activation_quant_key + self.weight_quant_key = weight_quant_key + @classmethod @abstractmethod def get_min_capability(cls) -> int: diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a8_mxfp4_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a8_mxfp4_fp8.py index e92ebf2508..0f0b2065a8 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a8_mxfp4_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w4a8_mxfp4_fp8.py @@ -3,7 +3,6 @@ from collections.abc import Callable from fractions import Fraction -from typing import Any import torch import torch.nn.functional as F @@ -11,7 +10,10 @@ import torch.nn.functional as F from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops from vllm.logger import init_logger from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, get_fp8_min_max, + kFp8StaticTensorSym, + kMxfp4Static, ) from vllm.model_executor.parameter import ( GroupQuantScaleParameter, @@ -38,39 +40,28 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme): Uses the AITER Triton kernel and falls back to emulation if AITER not available. """ + supported_activation_quant_keys = [kFp8StaticTensorSym] + supported_weight_quant_keys = [kMxfp4Static] + def __init__( self, - weight_quant_spec: dict[str, Any], - input_quant_spec: dict[str, Any], + activation_quant_key: QuantKey | None, ): + super().__init__(kMxfp4Static, activation_quant_key) self.out_dtype = None self.weight_dtype = "mxfp4" self.packed_factor: Fraction = Fraction(2, 1) # 2 FP4 values per byte self.weight_block_size = OCP_MX_BLOCK_SIZE - self.is_static_input_scheme = not input_quant_spec.get("is_dynamic") - self.input_qscheme = input_quant_spec.get("qscheme") # "per_tensor" - self.fp8_min, self.fp8_max = get_fp8_min_max() self.fp8_dtype = current_platform.fp8_dtype() - if not self.is_static_input_scheme: - raise NotImplementedError( - "Dynamic FP8 activation quantization is not yet supported " - "for W4A8. The current implementation expects static per-tensor " - "FP8 scales stored in the checkpoint." - ) - kernel_supported_gpu = False if current_platform.is_rocm(): kernel_supported_gpu = current_platform.supports_mx() - self.use_aiter_kernel = ( - is_aiter_found_and_supported() - and self.is_static_input_scheme - and kernel_supported_gpu - ) + self.use_aiter_kernel = is_aiter_found_and_supported() and kernel_supported_gpu if not self.use_aiter_kernel: logger.warning_once( @@ -128,17 +119,16 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme): layer.register_parameter("weight_scale", weight_scale) # INPUT SCALE (FP8 per-tensor static scale) - if self.is_static_input_scheme: - input_scale = PerTensorScaleParameter( - data=torch.empty( - len(output_partition_sizes), - dtype=torch.float32, - ), - weight_loader=weight_loader, - ) - # Initialize to avoid NaN - input_scale[:] = torch.finfo(torch.float32).min - layer.register_parameter("input_scale", input_scale) + input_scale = PerTensorScaleParameter( + data=torch.empty( + len(output_partition_sizes), + dtype=torch.float32, + ), + weight_loader=weight_loader, + ) + # Initialize to avoid NaN + input_scale[:] = torch.finfo(torch.float32).min + layer.register_parameter("input_scale", input_scale) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: # Ensuring weights & scales are non-trainable @@ -147,16 +137,15 @@ class QuarkW4A8_MXFP4_FP8(QuarkScheme): layer.weight_scale.data, requires_grad=False ) - if self.is_static_input_scheme: - input_scale = layer.input_scale.data - # For fused modules (QKV), take the max scale - if input_scale.numel() != 1: - input_scale = input_scale.max() + input_scale = layer.input_scale.data + # For fused modules (QKV), take the max scale + if input_scale.numel() != 1: + input_scale = input_scale.max() - layer.input_scale = torch.nn.Parameter( - torch.tensor(input_scale, dtype=torch.float32), - requires_grad=False, - ) + layer.input_scale = torch.nn.Parameter( + torch.tensor(input_scale, dtype=torch.float32), + requires_grad=False, + ) def apply_weights( self, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py index f7a9861e1d..40bdac7ee2 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Callable -from typing import Any, cast import torch from torch.nn import Parameter @@ -18,8 +17,11 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, + QuantKey, kFp8Dynamic128Sym, + kFp8DynamicTensorSym, kFp8DynamicTokenSym, + kFp8Static128BlockE8M0Sym, kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, @@ -42,36 +44,26 @@ logger = init_logger(__name__) class QuarkW8A8Fp8(QuarkScheme): + supported_activation_quant_keys: list[QuantKey | None] = [ + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8StaticTensorSym, + ] + supported_weight_quant_keys: list[QuantKey] = [ + kFp8StaticChannelSym, + kFp8StaticTensorSym, + ] + def __init__( - self, weight_config: dict[str, Any], input_config: dict[str, Any] | None + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - self.weight_qscheme = cast(str, weight_config.get("qscheme")) - self.is_static_input_scheme: bool = False - self.input_qscheme: str | None = None - if input_config is not None: - self.is_static_input_scheme = not cast(bool, input_config.get("is_dynamic")) - self.input_qscheme = cast(str, input_config.get("qscheme")) - - if self.weight_qscheme == "per_block": - raise ValueError( - "Quark W8A8 FP8 per-block weight quantization should use " - "QuarkW8A8Fp8PerBlock." - ) - per_token_activation = ( - not self.is_static_input_scheme and self.input_qscheme == "per_channel" - ) - per_channel_weight = self.weight_qscheme == "per_channel" - - self.activation_quant_key = ( - kFp8DynamicTokenSym if per_token_activation else kFp8StaticTensorSym - ) - # A per-output-channel weight scale is one fp32 value per weight row - # (length N). Tag it as ``GroupShape.PER_CHANNEL`` to match the - # canonical compressed-tensors CHANNEL strategy, so kernel selection - # (e.g. AITER's pre-shuffled FP8 GEMM) treats it uniformly. - self.weight_quant_key = ( - kFp8StaticChannelSym if per_channel_weight else kFp8StaticTensorSym + super().__init__(weight_quant_key, activation_quant_key) + self.weight_qscheme = ( + "per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor" ) + self.is_static_input_scheme = activation_quant_key == kFp8StaticTensorSym self.out_dtype = torch.get_default_dtype() self.input_dtype = get_current_vllm_config().model_config.dtype @@ -122,6 +114,8 @@ class QuarkW8A8Fp8(QuarkScheme): layer.input_scale = Parameter(input_scale, requires_grad=False) else: weight_scale = layer.weight_scale.data + + assert self.activation_quant_key is not None if self.activation_quant_key.scale.group_shape == GroupShape.PER_TOKEN: weight_scale = weight_scale.view(-1, 1) layer.weight = Parameter(weight.t(), requires_grad=False) @@ -191,6 +185,7 @@ class QuarkW8A8Fp8(QuarkScheme): input_scale[:] = torch.finfo(torch.float32).min layer.register_parameter("input_scale", input_scale) + assert self.activation_quant_key is not None self.fp8_linear = init_fp8_linear_kernel( activation_quant_key=self.activation_quant_key, weight_quant_key=self.weight_quant_key, @@ -210,22 +205,23 @@ class QuarkW8A8Fp8(QuarkScheme): class QuarkW8A8Fp8PerBlock(QuarkScheme): + supported_activation_quant_keys = [kFp8Dynamic128Sym] + supported_weight_quant_keys = [ + kFp8Static128BlockSym, + kFp8Static128BlockE8M0Sym, + ] + def __init__( - self, weight_config: dict[str, Any], input_config: dict[str, Any] | None + self, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - self.weight_qscheme = cast(str, weight_config.get("qscheme")) - if self.weight_qscheme != "per_block": - raise ValueError("QuarkW8A8Fp8PerBlock requires per-block weights.") - block_size = weight_config.get("block_size") - if not block_size: - raise ValueError( - "Quark W8A8 FP8 per-block weight quantization requires " - "`block_size` in the weight quantization config." - ) - self.weight_config = weight_config - self.weight_block_size = list(block_size) - self.activation_quant_key = kFp8Dynamic128Sym - self.weight_quant_key = kFp8Static128BlockSym + super().__init__(weight_quant_key, activation_quant_key) + self.weight_scale_dtype = weight_quant_key.scale.dtype + if self.weight_quant_key == kFp8Static128BlockE8M0Sym: + # TODO: oracle needs to support kFp8Static128BlockE8M0Sym. + self.weight_quant_key = kFp8Static128BlockSym + self.weight_block_size = list(weight_quant_key.scale.group_shape) self.out_dtype = torch.get_default_dtype() self.input_dtype = get_current_vllm_config().model_config.dtype @@ -272,8 +268,8 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme): layer.register_parameter("weight", weight) scale_dtype = ( - torch.float8_e8m0fnu - if self.weight_config.get("scale_type") == "float8_e8m0fnu" + self.weight_scale_dtype + if self.weight_scale_dtype != torch.float32 else None ) weight_scale = create_fp8_scale_parameter( @@ -287,6 +283,7 @@ class QuarkW8A8Fp8PerBlock(QuarkScheme): # DeepSeek V4 weight mappers route checkpoint ".scale" tensors here. layer.register_parameter("weight_scale_inv", weight_scale) + assert self.activation_quant_key is not None self.fp8_linear = init_fp8_linear_kernel( activation_quant_key=self.activation_quant_key, weight_quant_key=self.weight_quant_key, diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py index 1f0c5cbffd..713d80f3a8 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py @@ -10,6 +10,16 @@ from vllm.model_executor.kernels.linear import ( init_int8_linear_kernel, ) from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt8DynamicTensorAsym, + kInt8DynamicTensorSym, + kInt8DynamicTokenAsym, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, + kInt8StaticTensorAsym, + kInt8StaticTensorSym, +) from vllm.model_executor.parameter import ( BasevLLMParameter, ChannelQuantScaleParameter, @@ -21,15 +31,32 @@ logger = init_logger(__name__) class QuarkW8A8Int8(QuarkScheme): + supported_activation_quant_keys = [ + kInt8StaticTensorSym, + kInt8StaticTensorAsym, + kInt8DynamicTensorSym, + kInt8DynamicTensorAsym, + kInt8DynamicTokenSym, + kInt8DynamicTokenAsym, + ] + supported_weight_quant_keys = [ + kInt8StaticChannelSym, + kInt8StaticTensorSym, + ] + def __init__( self, - qscheme: str, - is_static_input_scheme: bool | None, - input_symmetric: bool | None, + weight_quant_key: QuantKey, + activation_quant_key: QuantKey | None, ): - self.qscheme = qscheme - self.is_static_input_scheme = is_static_input_scheme - self.input_symmetric = input_symmetric + super().__init__(weight_quant_key, activation_quant_key) + self.qscheme = ( + "per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor" + ) + + assert activation_quant_key is not None + self.is_static_input_scheme = activation_quant_key.scale.static + self.input_symmetric = activation_quant_key.symmetric @classmethod def get_min_capability(cls) -> int: diff --git a/vllm/model_executor/layers/quantization/quark/utils.py b/vllm/model_executor/layers/quantization/quark/utils.py index 8f6ce4d91c..7105561949 100644 --- a/vllm/model_executor/layers/quantization/quark/utils.py +++ b/vllm/model_executor/layers/quantization/quark/utils.py @@ -8,6 +8,8 @@ from typing import Any import regex as re import torch +QuarkQTensorHint = dict[str, Any] | list[dict[str, Any]] | None + def deep_compare(dict1: Any, dict2: Any) -> bool: if type(dict1) is not type(dict2): diff --git a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py index 0d56f179b9..af47ea2213 100644 --- a/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py +++ b/vllm/model_executor/layers/quantization/utils/ocp_mx_utils.py @@ -3,11 +3,32 @@ from enum import Enum from vllm.logger import init_logger +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kMxfp4Dynamic, + kMxfp4Static, + kMxfp6E2M3Dynamic, + kMxfp6E2M3Static, + kMxfp6E3M2Dynamic, + kMxfp6E3M2Static, +) logger = init_logger(__name__) OCP_MX_BLOCK_SIZE = 32 +_WEIGHT_QUANT_KEY_MAP: dict[str, QuantKey] = { + "mxfp4": kMxfp4Static, + "mxfp6_e3m2": kMxfp6E3M2Static, + "mxfp6_e2m3": kMxfp6E2M3Static, +} + +_ACTIVATION_QUANT_KEY_MAP: dict[str, QuantKey] = { + "mxfp4": kMxfp4Dynamic, + "mxfp6_e3m2": kMxfp6E3M2Dynamic, + "mxfp6_e2m3": kMxfp6E2M3Dynamic, +} + OCP_MX_DTYPES = { "mxfp4", "mxfp6_e3m2", diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index 20f33be240..c200fbbf78 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -222,6 +222,11 @@ kFp8Dynamic128Sym = QuantKey(FP8_DTYPE, kDynamic128Scale, symmetric=True) kStatic128BlockScale = ScaleDesc(torch.float32, True, GroupShape(128, 128)) kFp8Static128BlockSym = QuantKey(FP8_DTYPE, kStatic128BlockScale, symmetric=True) +kFp8Static128BlockE8M0Sym = QuantKey( + FP8_DTYPE, + ScaleDesc(torch.float8_e8m0fnu, True, GroupShape(128, 128)), + symmetric=True, +) kMxfp8StaticScale = ScaleDesc(torch.uint8, True, GroupShape(1, 32)) kMxfp8Static = QuantKey(FP8_DTYPE, kMxfp8StaticScale, symmetric=True) @@ -284,6 +289,9 @@ kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True) kInt8StaticTensorSym = QuantKey(torch.int8, kStaticTensorScale, symmetric=True) kInt8DynamicTensorSym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=True) +kInt8DynamicTokenAsym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=False) +kInt8StaticTensorAsym = QuantKey(torch.int8, kStaticTensorScale, symmetric=False) +kInt8DynamicTensorAsym = QuantKey(torch.int8, kDynamicTensorScale, symmetric=False) # INT4 W4A8 quantization keys