From e41a17e606c9a8ffa639ebde5bce885667ff27ea Mon Sep 17 00:00:00 2001 From: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:30:26 +0800 Subject: [PATCH] [ROCm][Quantization] Support AMD Quark per-block FP8 for fused MoE layers (#52263) Signed-off-by: huji Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> --- tests/quantization/test_quark.py | 159 +++++++++++++++++- .../compressed_tensors_moe_w8a8_fp8.py | 24 +-- .../model_executor/layers/quantization/fp8.py | 34 ++-- .../layers/quantization/quark/quark_moe.py | 93 ++++++++-- .../layers/quantization/utils/fp8_utils.py | 29 ++++ 5 files changed, 283 insertions(+), 56 deletions(-) diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 48c9b28321..ae236d71f8 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -11,7 +11,7 @@ import importlib.metadata from dataclasses import dataclass from importlib.util import find_spec from types import SimpleNamespace -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch import huggingface_hub import lm_eval @@ -26,6 +26,7 @@ from vllm.config.cache import CacheConfig from vllm.forward_context import set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( + FusedMoeWeightScaleSupported, RoutedExperts, UnquantizedFusedMoEMethod, ) @@ -48,6 +49,7 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501 from vllm.model_executor.layers.quantization.quark.quark_moe import ( # noqa: E501 QuarkMoEMethod, QuarkW4A8Fp8MoEMethod, + QuarkW8A8Fp8MoEMethod, QuarkW8A8Int8MoEMethod, ) from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme @@ -247,6 +249,26 @@ QTENSOR_CONFIGS = [ act_quant_key=kFp8Dynamic128Sym, dispatch_cls=QuarkW8A8Fp8PerBlock, ), + QTensorConfig( + name="fp8_w8a8_dynamic_block_fp32_moe", + weight={ + "dtype": "fp8_e4m3", + "qscheme": "per_block", + "is_dynamic": False, + "block_size": [128, 128], + "symmetric": True, + }, + input_tensors={ + "dtype": "fp8_e4m3", + "qscheme": "per_group", + "is_dynamic": True, + "group_size": 128, + "symmetric": True, + }, + weight_quant_key=kFp8Static128BlockSym, + act_quant_key=kFp8Dynamic128Sym, + dispatch_cls=QuarkW8A8Fp8MoEMethod, + ), QTensorConfig( name="fp8_w8a8_block_static_input", weight={ @@ -968,12 +990,111 @@ def test_quant_method_dispatch_instantiation(case, monkeypatch, default_vllm_con lambda: True, ) + # default_vllm_config carries no model, but the FP8 and OCP MX methods + # read the model type off the HF config. + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.quark.quark_moe." + "get_current_vllm_config", + lambda: SimpleNamespace( + model_config=SimpleNamespace(hf_config=SimpleNamespace()) + ), + ) + layer = TestRoutedExperts() method = config.get_quant_method(layer, "experts") assert isinstance(method, case.dispatch_cls) +QUARK_MOE_MODULE = "vllm.model_executor.layers.quantization.quark.quark_moe" +# validate_fp8_block_shape_moe imports this from vllm.distributed when called, +# so the patch has to target the source module rather than a local binding. +TP_WORLD_SIZE = "vllm.distributed.get_tensor_model_parallel_world_size" + + +def _make_per_block_fp8_moe_method( + activation_quant_key: QuantKey = kFp8Dynamic128Sym, +) -> QuarkW8A8Fp8MoEMethod: + return QuarkW8A8Fp8MoEMethod( + _make_test_moe_config(), + kFp8Static128BlockSym, + activation_quant_key, + ) + + +def test_quark_w8a8_fp8_moe_per_block_requires_dynamic_group_input(): + with ( + patch( + f"{QUARK_MOE_MODULE}.select_fp8_moe_backend", + return_value=(Mock(), Mock()), + ), + patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"), + pytest.raises(ValueError, match="per-block scales"), + ): + _make_per_block_fp8_moe_method(kFp8StaticTensorSym) + + +def test_quark_w8a8_fp8_moe_per_block_weight_shapes(): + with ( + patch( + f"{QUARK_MOE_MODULE}.select_fp8_moe_backend", + return_value=(Mock(), Mock()), + ), + patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"), + patch(TP_WORLD_SIZE, return_value=1), + ): + method = _make_per_block_fp8_moe_method() + assert method.block_quant + assert method.weight_block_size == [128, 128] + + layer = torch.nn.Module() + method.create_weights( + layer, + num_experts=4, + hidden_size=512, + intermediate_size_per_partition=256, + params_dtype=torch.bfloat16, + ) + + w13_num_shards = method.moe.w13_num_shards + assert layer.weight_block_size == [128, 128] + assert layer.w13_weight.shape == (4, w13_num_shards * 256, 512) + assert layer.w2_weight.shape == (4, 512, 256) + # Quark exports block scales as `weight_scale`, like the per-tensor and + # per-channel schemes, so all schemes register the same parameter name. + assert not hasattr(layer, "w13_weight_scale_inv") + assert not hasattr(layer, "w2_weight_scale_inv") + # One scale per 128x128 tile of each expert's weight. + assert layer.w13_weight_scale.shape == ( + 4, + w13_num_shards * (256 // 128), + 512 // 128, + ) + assert layer.w2_weight_scale.shape == (4, 512 // 128, 256 // 128) + # The loader shards block scales on the block grid, not per row. + for scale in (layer.w13_weight_scale, layer.w2_weight_scale): + assert scale.quant_method == FusedMoeWeightScaleSupported.BLOCK.value + + +def test_quark_w8a8_fp8_moe_per_block_rejects_misaligned_partition(): + with ( + patch( + f"{QUARK_MOE_MODULE}.select_fp8_moe_backend", + return_value=(Mock(), Mock()), + ), + patch(f"{QUARK_MOE_MODULE}.get_current_vllm_config"), + patch(TP_WORLD_SIZE, return_value=1), + pytest.raises(ValueError, match="not divisible by"), + ): + _make_per_block_fp8_moe_method().create_weights( + torch.nn.Module(), + num_experts=4, + hidden_size=512, + intermediate_size_per_partition=192, + params_dtype=torch.bfloat16, + ) + + @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) def test_quark_fp8_w_per_tensor_a_per_tensor( kv_cache_dtype: str, monkeypatch, dist_init, workspace_init @@ -1111,6 +1232,42 @@ def test_quark_int8_w8a8_moe(vllm_runner, tp): assert output +@pytest.mark.parametrize("tp", [1]) +def test_quark_fp8_w8a8_per_block_moe(vllm_runner, tp): + """Test per-block (128x128) FP8 MoE quantization with a tiny Qwen3 MoE model.""" + model_path = "Adamji/tiny-qwen3-moe-fp8-per-block" + with vllm_runner( + model_path, + enforce_eager=True, + tensor_parallel_size=tp, + gpu_memory_utilization=0.1, + ) as llm: + + def check_model(model): + experts = model.model.layers[0].mlp.experts + method = experts._quant_method + assert isinstance(method, QuarkW8A8Fp8MoEMethod), ( + f"Expected QuarkW8A8Fp8MoEMethod, got {type(method)}" + ) + assert method.weight_qscheme == "per_block" + assert method.weight_block_size == [128, 128] + + # hidden_size=128, moe_intermediate_size=256 and 4 experts, so one + # scale per 128x128 tile, with w13 stacking gate on top of up. + routed_experts = experts.routed_experts + assert routed_experts.w13_weight_scale.shape == (4, 4, 1) + assert routed_experts.w2_weight_scale.shape == (4, 1, 2) + # Quark exports the block scales under the same name as the + # per-tensor and per-channel schemes. + assert not hasattr(routed_experts, "w13_weight_scale_inv") + assert not hasattr(routed_experts, "w2_weight_scale_inv") + + llm.apply_model(check_model) + + output = llm.generate_greedy("Hello", max_tokens=4) + assert output + + @pytest.mark.skipif( not (on_gfx950() or on_gfx942()), reason="Quark W4A8 (INT4-FP8) MoE requires the AITER kernel on gfx942/gfx950", diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 7829ef65c0..262cd307a8 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -8,7 +8,6 @@ from compressed_tensors.quantization import ( QuantizationStrategy, ) -from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoeWeightScaleSupported, @@ -31,6 +30,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso from vllm.model_executor.layers.quantization.utils.fp8_utils import ( process_fp8_input_tensor_strategy_moe, process_fp8_weight_tensor_strategy_moe, + validate_fp8_block_shape_moe, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Dynamic128Sym, @@ -129,28 +129,14 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): if self.block_quant: assert self.weight_block_size is not None layer.weight_block_size = self.weight_block_size - tp_size = get_tensor_model_parallel_world_size() block_n, block_k = ( self.weight_block_size[0], self.weight_block_size[1], ) - # NOTE: To ensure proper alignment of the block-wise quantization - # scales, the output_size of the weights for both the gate and up - # layers must be divisible by block_n. - # Required by column parallel or enabling merged weights - if intermediate_size_per_partition % block_n != 0: - raise ValueError( - f"The output_size of gate's and up's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_n = {block_n}." - ) - if tp_size > 1 and intermediate_size_per_partition % block_k != 0: - # Required by row parallel - raise ValueError( - f"The input_size of down's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_k = {block_k}." - ) + validate_fp8_block_shape_moe( + intermediate_size_per_partition, + self.weight_block_size, + ) # WEIGHTS w13_weight = torch.nn.Parameter( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 2cf7168830..d8bef7347d 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -53,6 +53,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import ( process_fp8_weight_tensor_strategy, process_fp8_weight_tensor_strategy_moe, validate_fp8_block_shape, + validate_fp8_block_shape_moe, ) from vllm.model_executor.layers.quantization.utils.marlin_utils import ( get_marlin_input_dtype, @@ -575,34 +576,23 @@ class Fp8MoEMethod(FusedMoEMethodBase): assert self.moe_block_shape is not None moe_block_shape = self.moe_block_shape layer.weight_block_size = self.weight_block_size - tp_size = get_tensor_model_parallel_world_size() block_n, block_k = ( self.weight_block_size[0], self.weight_block_size[1], ) - # NOTE: To ensure proper alignment of the block-wise quantization - # scales, the output_size of the weights for both the gate and up - # layers must be divisible by block_n. - # Required by column parallel or enabling merged weights - if intermediate_size_per_partition % block_n != 0: - if self.weight_scale_refine is None: - raise ValueError( - f"The output_size of gate's and up's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_n = {block_n}." - ) + if self.weight_scale_refine is None: + validate_fp8_block_shape_moe( + intermediate_size_per_partition, + self.weight_block_size, + ) + else: # Use the refined block grid for the scale parameters; the # loader upsamples the checkpoint scales accordingly. - block_n, block_k = moe_block_shape - if tp_size > 1 and intermediate_size_per_partition % block_k != 0: - # Required by row parallel - if self.weight_scale_refine is None: - raise ValueError( - f"The input_size of down's weight = " - f"{intermediate_size_per_partition} is not divisible by " - f"weight quantization block_k = {block_k}." - ) - block_n, block_k = moe_block_shape + tp_size = get_tensor_model_parallel_world_size() + if intermediate_size_per_partition % block_n != 0: + block_n, block_k = moe_block_shape + if tp_size > 1 and intermediate_size_per_partition % block_k != 0: + block_n, block_k = moe_block_shape # WEIGHTS w13_weight = torch.nn.Parameter( diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index fd3fcc7e04..735499d18c 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -57,6 +57,9 @@ from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( select_nvfp4_moe_backend, ) from vllm.model_executor.layers.quantization.quark.utils import QuarkQTensorHint +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + validate_fp8_block_shape_moe, +) from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( _ACTIVATION_QUANT_KEY_MAP, _WEIGHT_QUANT_KEY_MAP, @@ -66,8 +69,10 @@ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, QuantKey, + kFp8Dynamic128Sym, kFp8DynamicTensorSym, kFp8DynamicTokenSym, + kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, kInt4W4A8StaticChannelSym, @@ -230,10 +235,12 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): kFp8StaticTensorSym, kFp8DynamicTensorSym, kFp8DynamicTokenSym, + kFp8Dynamic128Sym, ] supported_weight_quant_keys = [ kFp8StaticChannelSym, kFp8StaticTensorSym, + kFp8Static128BlockSym, ] def __init__( @@ -244,14 +251,18 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): ): super().__init__(moe, weight_quant_key, activation_quant_key) self.weight_dtype = "fp8" - self.weight_qscheme = ( - "per_channel" if weight_quant_key == kFp8StaticChannelSym else "per_tensor" - ) - self.input_qscheme = ( - "per_channel" - if activation_quant_key == kFp8DynamicTokenSym - else "per_tensor" - ) + if weight_quant_key == kFp8StaticChannelSym: + self.weight_qscheme = "per_channel" + elif weight_quant_key == kFp8Static128BlockSym: + self.weight_qscheme = "per_block" + else: + self.weight_qscheme = "per_tensor" + if activation_quant_key == kFp8DynamicTokenSym: + self.input_qscheme = "per_channel" + elif activation_quant_key == kFp8Dynamic128Sym: + self.input_qscheme = "per_group" + else: + self.input_qscheme = "per_tensor" self.static_input_scales = activation_quant_key == kFp8StaticTensorSym per_tensor = ( self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor" @@ -259,16 +270,31 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): per_channel = ( self.weight_qscheme == "per_channel" and self.input_qscheme == "per_channel" ) - self.act_quant_group_shape = ( - GroupShape.PER_TOKEN if per_channel else GroupShape.PER_TENSOR + per_block = ( + self.weight_qscheme == "per_block" and self.input_qscheme == "per_group" ) - if not (per_tensor or per_channel): + if not (per_tensor or per_channel or per_block): raise ValueError( - "For FP8 Fused MoE layers, only per-tensor and per-channel " - "scales for weights and activations are supported. Found " - f"{self.weight_qscheme}, {self.input_qscheme}" + "For FP8 Fused MoE layers, only per-tensor, per-channel and " + "per-block scales for weights and activations are supported. " + f"Found {self.weight_qscheme}, {self.input_qscheme}" ) # noqa E501 + # One scale per (block_n, block_k) tile of the weight, so the block + # shape is carried by the weight key's group shape. + self.weight_block_size: list[int] | None = ( + list(weight_quant_key.scale.group_shape) if per_block else None + ) + self.block_quant = self.weight_block_size is not None + + if per_channel: + self.act_quant_group_shape = GroupShape.PER_TOKEN + elif per_block: + assert self.weight_block_size is not None + self.act_quant_group_shape = GroupShape(1, self.weight_block_size[1]) + else: + self.act_quant_group_shape = GroupShape.PER_TENSOR + self.fp8_backend, self.experts_cls = select_fp8_moe_backend( config=moe, weight_key=self.weight_quant_key, @@ -293,6 +319,15 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): layer.weight_block_size = None params_dtype = torch.float8_e4m3fn + if self.block_quant: + assert self.weight_block_size is not None + layer.weight_block_size = self.weight_block_size + block_n, block_k = self.weight_block_size[0], self.weight_block_size[1] + validate_fp8_block_shape_moe( + intermediate_size_per_partition, + self.weight_block_size, + ) + # WEIGHTS w13_weight = torch.nn.Parameter( torch.zeros( @@ -368,6 +403,35 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): ) set_weight_attrs(w13_weight_scale, extra_weight_attrs) set_weight_attrs(w2_weight_scale, extra_weight_attrs) + elif self.weight_qscheme == "per_block": + # One scale per (block_n, block_k) tile of each expert's weight. + w13_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + self.moe.w13_num_shards + * ((intermediate_size_per_partition + block_n - 1) // block_n), + (hidden_size + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + w2_weight_scale = torch.nn.Parameter( + torch.ones( + num_experts, + (hidden_size + block_n - 1) // block_n, + (intermediate_size_per_partition + block_k - 1) // block_k, + dtype=torch.float32, + ), + requires_grad=False, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + # Add PER-BLOCK quantization for RoutedExperts.weight_loader. + extra_weight_attrs.update( + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value} + ) + set_weight_attrs(w13_weight_scale, extra_weight_attrs) + set_weight_attrs(w2_weight_scale, extra_weight_attrs) # INPUT_SCALES if self.static_input_scales: @@ -548,6 +612,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): a2_scale=layer.w2_input_scale, w1_bias=getattr(layer, "w13_bias", None), w2_bias=getattr(layer, "w2_bias", None), + block_shape=self.weight_block_size, per_act_token_quant=self.input_qscheme == "per_channel", per_out_ch_quant=self.weight_qscheme == "per_channel", swiglu_limit=getattr(layer, "swiglu_limit", None), diff --git a/vllm/model_executor/layers/quantization/utils/fp8_utils.py b/vllm/model_executor/layers/quantization/utils/fp8_utils.py index 451ae3a49a..199d856e3b 100644 --- a/vllm/model_executor/layers/quantization/utils/fp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/fp8_utils.py @@ -1265,6 +1265,35 @@ def validate_fp8_block_shape( ) +def validate_fp8_block_shape_moe( + intermediate_size_per_partition: int, + block_size: list[int], +) -> None: + """Validate fused MoE block quantization shapes for tensor parallelism.""" + from vllm.distributed import get_tensor_model_parallel_world_size + + tp_size = get_tensor_model_parallel_world_size() + block_n, block_k = block_size[0], block_size[1] + + # NOTE: To ensure proper alignment of the block-wise quantization + # scales, the output_size of the weights for both the gate and up + # layers must be divisible by block_n. + # Required by column parallel or enabling merged weights + if intermediate_size_per_partition % block_n != 0: + raise ValueError( + f"The output_size of gate's and up's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_n = {block_n}." + ) + if tp_size > 1 and intermediate_size_per_partition % block_k != 0: + # Required by row parallel + raise ValueError( + f"The input_size of down's weight = " + f"{intermediate_size_per_partition} is not divisible by " + f"weight quantization block_k = {block_k}." + ) + + def create_fp8_weight_parameter( output_size_per_partition: int, input_size_per_partition: int,