diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index a0319ab20d..f3e1eea1d7 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -104,11 +104,20 @@ QWEN3_AUTOROUND_MODELS = [ ), id="auto_round:mxfp4:qwen3-30b-a3b", ), + pytest.param( + "INCModel/Qwen3-30B-A3B-12L-MXFP8-test", + marks=pytest.mark.skipif( + not (current_platform.is_cuda() or current_platform.is_xpu()), + reason="Qwen3-30B-A3B MXFP8 AutoRound model requires CUDA/XPU.", + ), + id="auto_round:mxfp8:qwen3-30b-a3b", + ), ] MODEL_RUNNER_KWARGS: dict[str, dict[str, Any]] = { "INCModel/Qwen3-1.7B-AutoRound-MXFP4-W4A4": {"enforce_eager": True}, "INCModel/Qwen3-30B-A3B-12L-MXFP4-test": {"enforce_eager": True}, + "INCModel/Qwen3-30B-A3B-12L-MXFP8-test": {"enforce_eager": True}, "Intel/Qwen3-8B-w2g64-for-ut": { "block_size": 64, "gpu_memory_utilization": 0.8, @@ -232,6 +241,23 @@ def make_qwen3_autoround_config(kind: str) -> INCConfig: }, }, }, + "qwen3_30b_a3b_mxfp8": { + "quant_method": "auto-round", + "bits": 8, + "group_size": 32, + "sym": True, + "packing_format": "auto_round:llm_compressor", + "data_type": "mx_fp", + "act_bits": 8, + "act_group_size": 32, + "act_data_type": "mx_fp", + "extra_config": { + "model.layers.0.mlp.gate": { + "bits": 16, + "data_type": "float", + }, + }, + }, } try: config = configs[kind] @@ -606,6 +632,113 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe( assert method.moe_config is layer.moe_config +def test_qwen3_30b_a3b_mxfp8_autoround_routes_to_mxfp8_moe( + monkeypatch, +) -> None: + expected_backend = object() + expected_experts_cls = object() + captured = {} + + def fake_select_mxfp8_moe_backend(*, config): + captured["moe_config"] = config + return expected_backend, expected_experts_cls + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes." + "inc_mxfp8_moe.select_mxfp8_moe_backend", + fake_select_mxfp8_moe_backend, + ) + + config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp8") + layer = object.__new__(RoutedExperts) + layer.moe_config = cast(Any, "moe-config") + + linear_method = config.get_quant_method( + object.__new__(LinearBase), "model.layers.0.self_attn.q_proj" + ) + moe_method = config.get_quant_method(layer, "model.layers.0.mlp") + gate_method = config.get_quant_method( + object.__new__(LinearBase), "model.layers.0.mlp.gate" + ) + + from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import ( + INCMxfp8MoEMethod, + ) + + assert isinstance(linear_method, INCLinearMethod) + assert isinstance(linear_method.scheme, INCMxfp8LinearScheme) + assert isinstance(moe_method, INCMxfp8MoEMethod) + assert moe_method.mxfp8_backend is expected_backend + assert moe_method.experts_cls is expected_experts_cls + assert captured["moe_config"] == layer.moe_config + assert isinstance(gate_method, UnquantizedLinearMethod) + + +def test_inc_mxfp8_moe_method_registers_block_scaled_weights(monkeypatch) -> None: + expected_backend = object() + expected_experts_cls = object() + + monkeypatch.setattr( + "vllm.model_executor.layers.quantization.inc.schemes." + "inc_mxfp8_moe.select_mxfp8_moe_backend", + lambda *, config: (expected_backend, expected_experts_cls), + ) + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_rank", + lambda: 0, + ) + monkeypatch.setattr( + "vllm.model_executor.parameter.get_tensor_model_parallel_world_size", + lambda: 1, + ) + + from vllm.model_executor.layers.quantization.inc.schemes.inc_mxfp8_moe import ( + INCMxfp8MoEMethod, + ) + + class DummyMoeConfig: + is_act_and_mul = True + + method = INCMxfp8MoEMethod(cast(Any, DummyMoeConfig())) + layer = torch.nn.Module() + layer.hidden_size = 64 + layer.intermediate_size_per_partition = 32 + + method.create_weights( + layer, + num_experts=2, + hidden_size=64, + intermediate_size_per_partition=32, + params_dtype=torch.bfloat16, + ) + + assert method.mxfp8_backend is expected_backend + assert method.experts_cls is expected_experts_cls + assert method.weight_block_size == [1, 32] + assert layer.w13_weight.shape == (2, 64, 64) + assert layer.w2_weight.shape == (2, 64, 32) + assert layer.w13_weight.dtype is torch.float8_e4m3fn + assert layer.w2_weight.dtype is torch.float8_e4m3fn + assert layer.w13_weight_scale.shape == (2, 64, 2) + assert layer.w2_weight_scale.shape == (2, 64, 1) + assert layer.w13_weight_scale.dtype is torch.uint8 + assert layer.w2_weight_scale.dtype is torch.uint8 + assert layer.w13_weight_scale.quant_method == "block" + assert layer.w2_weight_scale.quant_method == "block" + + misaligned_layer = torch.nn.Module() + misaligned_layer.hidden_size = 65 + misaligned_layer.intermediate_size_per_partition = 32 + with pytest.raises(ValueError, match="hidden_size divisible by 32"): + method.create_weights( + misaligned_layer, + num_experts=2, + hidden_size=65, + intermediate_size_per_partition=32, + params_dtype=torch.bfloat16, + ) + + def test_inc_mxfp4_linear_method_registers_and_processes_weights( monkeypatch, ) -> None: diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py new file mode 100644 index 0000000000..2c9a06b798 --- /dev/null +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_moe.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.fused_moe import ( + FusedMoEQuantConfig, + FusedMoeWeightScaleSupported, + RoutedExperts, + SharedExperts, +) +from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( + FusedMoEMethodBase, +) +from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( + convert_to_fp8_moe_kernel_format, + make_fp8_moe_kernel, + make_fp8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( + select_mxfp8_moe_backend, +) +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + MXFP8_BLOCK_SIZE, + MXFP8_SCALE_DTYPE, + MXFP8_VALUE_DTYPE, +) +from vllm.model_executor.parameter import ModelWeightParameter +from vllm.model_executor.utils import replace_parameter, set_weight_attrs + + +class INCMxfp8MoEMethod(FusedMoEMethodBase): + """W8A8 MXFP8 MoE method for serialized AutoRound checkpoints.""" + + def __init__(self, moe) -> None: + super().__init__(moe) + self.weight_block_size = [1, MXFP8_BLOCK_SIZE] + self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe) + + def create_weights( + self, + layer: RoutedExperts, + num_experts: int, + hidden_size: int, + intermediate_size_per_partition: int, + params_dtype: torch.dtype, + **extra_weight_attrs, + ) -> None: + assert layer.intermediate_size_per_partition == intermediate_size_per_partition + assert layer.hidden_size == hidden_size + layer.orig_dtype = params_dtype + + if hidden_size % MXFP8_BLOCK_SIZE != 0: + raise ValueError( + f"MXFP8 MoE requires hidden_size divisible by {MXFP8_BLOCK_SIZE}, " + f"got {hidden_size}." + ) + if intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0: + raise ValueError( + "MXFP8 MoE requires intermediate_size_per_partition divisible by " + f"{MXFP8_BLOCK_SIZE}, got {intermediate_size_per_partition}." + ) + + layer.num_experts = num_experts + weight_loader = extra_weight_attrs.get("weight_loader") + w13_num_shards = 2 if self.moe.is_act_and_mul else 1 + + w13_weight = ModelWeightParameter( + data=torch.empty( + num_experts, + w13_num_shards * intermediate_size_per_partition, + hidden_size, + dtype=MXFP8_VALUE_DTYPE, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight", w13_weight) + + w2_weight = ModelWeightParameter( + data=torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition, + dtype=MXFP8_VALUE_DTYPE, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight", w2_weight) + + w13_weight_scale = ModelWeightParameter( + data=torch.empty( + num_experts, + w13_num_shards * intermediate_size_per_partition, + hidden_size // MXFP8_BLOCK_SIZE, + dtype=MXFP8_SCALE_DTYPE, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w13_weight_scale", w13_weight_scale) + + w2_weight_scale = ModelWeightParameter( + data=torch.empty( + num_experts, + hidden_size, + intermediate_size_per_partition // MXFP8_BLOCK_SIZE, + dtype=MXFP8_SCALE_DTYPE, + ), + input_dim=2, + output_dim=1, + weight_loader=weight_loader, + ) + layer.register_parameter("w2_weight_scale", w2_weight_scale) + + set_weight_attrs( + layer.w13_weight_scale, + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}, + ) + set_weight_attrs( + layer.w2_weight_scale, + {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}, + ) + + def process_weights_after_loading(self, layer: RoutedExperts) -> None: + layer.weight_block_size = self.weight_block_size + + w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format( + fp8_backend=self.mxfp8_backend, + layer=layer, + w13=layer.w13_weight, + w2=layer.w2_weight, + w13_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + w13_input_scale=None, + w2_input_scale=None, + ) + + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + replace_parameter(layer, "w13_weight_scale", w13_scale) + replace_parameter(layer, "w2_weight_scale", w2_scale) + + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.moe_quant_config is not None + self.moe_kernel = make_fp8_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + fp8_backend=self.mxfp8_backend, + experts_cls=self.experts_cls, + routing_tables=layer._expert_routing_tables(), + ) + + def get_fused_moe_quant_config( + self, layer: RoutedExperts + ) -> FusedMoEQuantConfig | None: + return make_fp8_moe_quant_config( + fp8_backend=self.mxfp8_backend, + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=None, + a2_scale=None, + block_shape=self.weight_block_size, + swiglu_limit=getattr(layer, "swiglu_limit", None), + gemm1_alpha=getattr(layer, "swiglu_alpha", None), + gemm1_beta=getattr(layer, "swiglu_beta", None), + layer=layer, + ) + + def apply( + self, + layer: RoutedExperts, + x: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + shared_experts: SharedExperts | None, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor: + assert self.moe_kernel is not None + return self.moe_kernel.apply( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights, + topk_ids, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + expert_map=layer.expert_map, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, + ) diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py index 2e251a8a54..70220a0f36 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp8_scheme.py @@ -29,3 +29,15 @@ class INCMxfp8Scheme(INCScheme): from .inc_mxfp8_linear import INCMxfp8LinearScheme return INCLinearMethod(INCMxfp8LinearScheme()) + + def get_moe_method( + self, + config: "INCConfig", + layer: "torch.nn.Module", + prefix: str, + layer_config: "INCLayerConfig", + ): + del config, prefix, layer_config + from .inc_mxfp8_moe import INCMxfp8MoEMethod + + return INCMxfp8MoEMethod(layer.moe_config)