From 7c77868cdf57c3f711ba971fa68d421b5b39a9c4 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Thu, 6 Aug 2026 08:24:47 +0800 Subject: [PATCH] [Bugfix] Size and iterate w13 by shard count for non-gated MoE (#51125) Signed-off-by: aoshen02 Co-authored-by: Matej Sirovatka Co-authored-by: Rishi Puri Co-authored-by: Hsiao-Yuan Chen Co-authored-by: Claude Opus 5 (1M context) --- tests/quantization/test_auto_gptq.py | 1 + tests/quantization/test_auto_round.py | 6 ++- .../model_executor/layers/fused_moe/config.py | 5 ++ .../layers/quantization/auto_awq.py | 10 ++-- .../layers/quantization/auto_gptq.py | 10 ++-- .../layers/quantization/bitsandbytes.py | 6 +-- .../compressed_tensors_moe_w4a4_mxfp4.py | 4 +- .../compressed_tensors_moe_w4a8_fp8.py | 6 +-- .../model_executor/layers/quantization/fp8.py | 17 ++++-- .../layers/quantization/humming.py | 4 +- .../quantization/inc/schemes/inc_mxfp4_moe.py | 4 +- .../layers/quantization/moe_wna16.py | 8 +-- .../layers/quantization/mxfp4.py | 24 ++++----- .../layers/quantization/online/fp8.py | 22 ++++---- .../layers/quantization/online/moe_base.py | 4 +- .../layers/quantization/quark/quark_moe.py | 53 ++++++++++--------- 16 files changed, 104 insertions(+), 80 deletions(-) diff --git a/tests/quantization/test_auto_gptq.py b/tests/quantization/test_auto_gptq.py index 9eefd88e91..b8507022bd 100644 --- a/tests/quantization/test_auto_gptq.py +++ b/tests/quantization/test_auto_gptq.py @@ -65,6 +65,7 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases(): method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {}) method.input_dtype = None method.experts_cls = None + method.moe = SimpleNamespace(w13_num_shards=2) layer = torch.nn.Module() method.create_weights( diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index eef3a91de5..86bd857923 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -8,6 +8,7 @@ Validating the configuration and printing results for manual checking. Run `pytest tests/quantization/test_auto_round.py`. """ +from types import SimpleNamespace from typing import Any, cast import pytest @@ -694,7 +695,8 @@ def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel( INCMxfp4MoEMethod, ) - method = INCMxfp4MoEMethod(moe=cast(Any, "moe-config")) + expected_moe_config = SimpleNamespace(w13_num_shards=2) + method = INCMxfp4MoEMethod(moe=cast(Any, expected_moe_config)) layer = torch.nn.Module() layer._expert_routing_tables = lambda: "routing-tables" @@ -723,7 +725,7 @@ def test_inc_mxfp4_moe_method_registers_weights_and_builds_kernel( assert captured["quant_config_kwargs"]["w1_scale"] is layer.w13_weight_scale assert captured["quant_config_kwargs"]["w2_scale"] is layer.w2_weight_scale assert captured["kernel_kwargs"]["moe_quant_config"] is expected_quant_config - assert captured["kernel_kwargs"]["moe_config"] == "moe-config" + assert captured["kernel_kwargs"]["moe_config"] is expected_moe_config assert captured["kernel_kwargs"]["experts_cls"] is expected_experts_cls assert captured["kernel_kwargs"]["routing_tables"] == "routing-tables" assert method.moe_kernel is expected_kernel diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index f2ee520d09..0c633090ad 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1375,6 +1375,11 @@ class FusedMoEConfig: def is_act_and_mul(self) -> bool: return self.activation.is_gated + @property + def w13_num_shards(self) -> int: + """Number of shards fused into w13: gate and up for gated, up only.""" + return 2 if self.is_act_and_mul else 1 + @property def tp_size(self): return self.moe_parallel_config.tp_size diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 5e9946ff35..4c062e7772 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -592,7 +592,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, hidden_size, - 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + self.moe.w13_num_shards + * intermediate_size_per_partition + // self.quant_config.pack_factor, dtype=torch.int32, ), requires_grad=False, @@ -623,7 +625,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, num_groups_w13, - intermediate_size_per_partition * 2, + intermediate_size_per_partition * self.moe.w13_num_shards, dtype=params_dtype, ), requires_grad=False, @@ -644,7 +646,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, num_groups_w13, - 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + self.moe.w13_num_shards + * intermediate_size_per_partition + // self.quant_config.pack_factor, dtype=torch.int32, ), requires_grad=False, diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index 710e947f0b..14b9aa94c8 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -541,7 +541,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, hidden_size // self.quant_config.pack_factor, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.int32, ), requires_grad=False, @@ -565,7 +565,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, scales_size13, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=params_dtype, ), requires_grad=False, @@ -586,7 +586,9 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): torch.empty( num_experts, scales_size13, - 2 * intermediate_size_per_partition // self.quant_config.pack_factor, + self.moe.w13_num_shards + * intermediate_size_per_partition + // self.quant_config.pack_factor, dtype=torch.int32, ), requires_grad=False, @@ -654,7 +656,7 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=params_dtype, ), requires_grad=False, diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py index 23aa321017..ec3162c08a 100644 --- a/vllm/model_executor/layers/quantization/bitsandbytes.py +++ b/vllm/model_executor/layers/quantization/bitsandbytes.py @@ -523,7 +523,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase): quant_ratio = calculate_quant_ratio(params_dtype) # Fused gate_up_proj (column parallel) w13_total_size = ( - hidden_size * 2 * intermediate_size_per_partition + hidden_size * self.moe.w13_num_shards * intermediate_size_per_partition ) // quant_ratio w13_qweight = torch.nn.Parameter( torch.empty( @@ -541,10 +541,10 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase): { "num_experts": num_experts, "input_dim": hidden_size, - "output_dim": 2 * intermediate_size_per_partition, + "output_dim": self.moe.w13_num_shards * intermediate_size_per_partition, "experts_shape": ( num_experts, - intermediate_size_per_partition * 2, + intermediate_size_per_partition * self.moe.w13_num_shards, hidden_size, ), "pack_factor": quant_ratio, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index 1a0cd8ed10..bb266c0f82 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -75,7 +75,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): w13_weight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, # 2 fp4 items are packed in the input dimension hidden_size // 2, requires_grad=False, @@ -102,7 +102,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, # 2 fp4 items are packed in the input dimension hidden_size // self.group_size, dtype=torch.uint8, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py index 224b1a62bf..dcf52464b1 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py @@ -79,7 +79,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): w13_weight_packed = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // self.packed_factor, dtype=params_dtype, ), @@ -107,7 +107,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // self.group_size, dtype=layer.orig_dtype, ), @@ -147,7 +147,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): w13_weight_chan_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index cff4c547df..98d126ce44 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -545,7 +545,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): w13_weight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size, dtype=params_dtype, ), @@ -571,7 +571,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=layer.orig_dtype, ), requires_grad=False, @@ -588,13 +588,16 @@ class Fp8MoEMethod(FusedMoEMethodBase): # WEIGHT_SCALES if not self.block_quant: # For per-tensor quant, the scales are per expert and weight. - w13_scale_data = torch.ones(num_experts, 2, dtype=torch.float32) + w13_scale_data = torch.ones( + num_experts, self.moe.w13_num_shards, dtype=torch.float32 + ) w2_scale_data = torch.ones(num_experts, dtype=torch.float32) else: # For block quant, the scales are per block (typically 128x128). w13_scale_data = torch.ones( num_experts, - 2 * ((intermediate_size_per_partition + block_n - 1) // block_n), + self.moe.w13_num_shards + * ((intermediate_size_per_partition + block_n - 1) // block_n), (hidden_size + block_k - 1) // block_k, dtype=torch.float32, ) @@ -722,7 +725,11 @@ class Fp8MoEMethod(FusedMoEMethodBase): if not self.block_quant: shard_size = layer.intermediate_size_per_partition w13, w13_scale = process_fp8_weight_tensor_strategy_moe( - w13, w13_scale, shard_size, layer.local_num_experts + w13, + w13_scale, + shard_size, + layer.local_num_experts, + is_act_and_mul=self.moe.is_act_and_mul, ) # Shuffle weights to runtime format and setup kernel. diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index f73852e0cf..4e1ae0d0e4 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -728,10 +728,10 @@ class HummingMoEMethod(FusedMoEMethodBase): # The weight names of sublayer start with the prefix "{sublayer_name}_" layer.sublayer_configs = { "w13": { - "shape_n": intermediate_size_per_partition * 2, + "shape_n": intermediate_size_per_partition * self.moe.w13_num_shards, "shape_k": hidden_size, "tensors_attrs": self.weight_schema.get_padded_tensors_attrs( - shape_n=intermediate_size_per_partition * 2, + shape_n=intermediate_size_per_partition * self.moe.w13_num_shards, shape_k=hidden_size, num_experts=num_experts, param_dtype=params_dtype, diff --git a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py index 7cb3adce09..437a744947 100644 --- a/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py +++ b/vllm/model_executor/layers/quantization/inc/schemes/inc_mxfp4_moe.py @@ -88,7 +88,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase): w13_weight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // 2, dtype=torch.uint8, ), @@ -113,7 +113,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase): w13_weight_scale = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // self.group_size, dtype=torch.uint8, ), diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index 76f2d740bc..1e54a627b6 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -276,7 +276,7 @@ class MoeWNA16Method(FusedMoEMethodBase): w13_qweight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // bit8_pack_factor, dtype=torch.uint8, ), @@ -301,7 +301,7 @@ class MoeWNA16Method(FusedMoEMethodBase): w13_scales = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // group_size, dtype=params_dtype, ), @@ -326,7 +326,9 @@ class MoeWNA16Method(FusedMoEMethodBase): w13_qzeros = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition // bit8_pack_factor, + self.moe.w13_num_shards + * intermediate_size_per_partition + // bit8_pack_factor, hidden_size // group_size, dtype=torch.uint8, ), diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 266508a005..47d47614c4 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -215,7 +215,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): w13_weight = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // 2, dtype=weight_dtype, ), @@ -227,7 +227,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): w13_weight_scale = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // mxfp4_block, dtype=scale_dtype, ), @@ -267,7 +267,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.bfloat16, ), requires_grad=False, @@ -305,13 +305,13 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): assert ( w13.dim() == 3 and w13.shape[0] == num_experts - and w13.shape[1] == intermediate_size * 2 + and w13.shape[1] == intermediate_size * self.moe.w13_num_shards and w13.shape[2] == hidden_size // 2 ) assert ( w13_scale.dim() == 3 and w13_scale.shape[0] == num_experts - and w13_scale.shape[1] == intermediate_size * 2 + and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards and w13_scale.shape[2] == hidden_size // sf_block_size ) assert ( @@ -329,7 +329,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): assert ( w13_bias.dim() == 2 and w13_bias.shape[0] == num_experts - and w13_bias.shape[1] == intermediate_size * 2 + and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards ) if w2_bias is not None: assert ( @@ -599,7 +599,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w13_weight = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // 2, dtype=weight_dtype, ), @@ -611,7 +611,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w13_weight_scale = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // mxfp4_block, dtype=scale_dtype, ), @@ -651,7 +651,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.bfloat16, ), requires_grad=False, @@ -689,13 +689,13 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): assert ( w13.dim() == 3 and w13.shape[0] == num_experts - and w13.shape[1] == intermediate_size * 2 + and w13.shape[1] == intermediate_size * self.moe.w13_num_shards and w13.shape[2] == hidden_size // 2 ) assert ( w13_scale.dim() == 3 and w13_scale.shape[0] == num_experts - and w13_scale.shape[1] == intermediate_size * 2 + and w13_scale.shape[1] == intermediate_size * self.moe.w13_num_shards and w13_scale.shape[2] == hidden_size // sf_block_size ) assert ( @@ -713,7 +713,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): assert ( w13_bias.dim() == 2 and w13_bias.shape[0] == num_experts - and w13_bias.shape[1] == intermediate_size * 2 + and w13_bias.shape[1] == intermediate_size * self.moe.w13_num_shards ) if w2_bias is not None: assert ( diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index e63c285aa3..1ec6836059 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -582,12 +582,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase): hidden_size = layer.moe_config.hidden_dim_unpadded intermediate_size = layer.moe_config.intermediate_size_per_partition_unpadded - w13_half_size = layer.w13_weight.shape[1] // 2 - if w13_half_size > intermediate_size: - layer.w13_weight[:, intermediate_size:w13_half_size, :] = 0 - layer.w13_weight[ - :, w13_half_size + intermediate_size : 2 * w13_half_size, : - ] = 0 + w13_shard = layer.w13_weight.shape[1] // self.moe.w13_num_shards + if w13_shard > intermediate_size: + for shard in range(self.moe.w13_num_shards): + start = shard * w13_shard + intermediate_size + layer.w13_weight[:, start : (shard + 1) * w13_shard, :] = 0 if layer.w13_weight.shape[2] > hidden_size: layer.w13_weight[:, :, hidden_size:] = 0 @@ -597,12 +596,11 @@ class Fp8PerBlockOnlineMoEMethod(_Fp8OnlineMoEBase): layer.w2_weight[:, :, intermediate_size:] = 0 if getattr(layer, "w13_bias", None) is not None: - w13_bias_half_size = layer.w13_bias.shape[1] // 2 - if w13_bias_half_size > intermediate_size: - layer.w13_bias[:, intermediate_size:w13_bias_half_size] = 0 - layer.w13_bias[ - :, w13_bias_half_size + intermediate_size : 2 * w13_bias_half_size - ] = 0 + w13_bias_shard = layer.w13_bias.shape[1] // self.moe.w13_num_shards + if w13_bias_shard > intermediate_size: + for shard in range(self.moe.w13_num_shards): + start = shard * w13_bias_shard + intermediate_size + layer.w13_bias[:, start : (shard + 1) * w13_bias_shard] = 0 if ( getattr(layer, "w2_bias", None) is not None diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index 27d3ccc307..6aced453a1 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -40,7 +40,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): w13_weight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size, device="meta", dtype=params_dtype, @@ -69,7 +69,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, device="meta", dtype=layer.orig_dtype, ), diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index c5c9c6988f..f1a22c8903 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -216,7 +216,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): w13_weight = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size, dtype=params_dtype, ), @@ -243,7 +243,10 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): # They are combined to a single scale after weight loading. if self.model_type != "gpt_oss": w13_weight_scale = torch.nn.Parameter( - torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + torch.ones( + num_experts, self.moe.w13_num_shards, dtype=torch.float32 + ), + requires_grad=False, ) else: # For gpt_oss, the w1(gate) & w3(up) are fused as one. @@ -267,7 +270,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -306,7 +309,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -400,7 +403,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod): # For non-gpt_oss, process w1 and w3 shards separately for expert_id in range(layer.local_num_experts): start = 0 - for shard_id in range(2): + for shard_id in range(self.moe.w13_num_shards): dq_weight = per_tensor_dequantize( layer.w13_weight[expert_id][start : start + shard_size, :], layer.w13_weight_scale[expert_id][shard_id], @@ -560,7 +563,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): w13_weight = torch.nn.Parameter( torch.empty( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size, dtype=params_dtype, ), @@ -586,7 +589,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -605,7 +608,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): else: # per-tensor: one scalar per expert (two for the fused w1/w3) w13_weight_scale = torch.nn.Parameter( - torch.ones(num_experts, 2, dtype=torch.float32), + torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32), requires_grad=False, ) layer.register_parameter("w13_weight_scale", w13_weight_scale) @@ -644,7 +647,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): # ZERO POINTS (loaded but discarded after loading; kernel uses symmetric) w13_input_zero_point = torch.nn.Parameter( - torch.zeros(num_experts, 2, dtype=torch.int8), + torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8), requires_grad=False, ) layer.register_parameter("w13_input_zero_point", w13_input_zero_point) @@ -661,7 +664,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): w13_weight_zero_point = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.int8, ), requires_grad=False, @@ -672,7 +675,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): ) else: w13_weight_zero_point = torch.nn.Parameter( - torch.zeros(num_experts, 2, dtype=torch.int8), + torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8), requires_grad=False, ) w2_weight_zero_point = torch.nn.Parameter( @@ -689,7 +692,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -762,7 +765,7 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): for expert_id in range(layer.local_num_experts): start = 0 - for shard_id in range(2): + for shard_id in range(self.moe.w13_num_shards): dq_weight = per_tensor_dequantize( layer.w13_weight[expert_id][start : start + shard_size, :], layer.w13_weight_scale[expert_id][shard_id], @@ -905,7 +908,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): w13_weight = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // 8, # INT32 packing for W4 dtype=params_dtype, ), @@ -927,7 +930,8 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): # Per-tensor fp8 weight scales w13_weight_scale = torch.nn.Parameter( - torch.ones(num_experts, 2, dtype=torch.float32), requires_grad=False + torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32), + requires_grad=False, ) w2_weight_scale = torch.nn.Parameter( torch.ones(num_experts, dtype=torch.float32), requires_grad=False @@ -944,7 +948,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): w13_weight_scale_2 = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -978,7 +982,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod): for expert_id in range(layer.local_num_experts): start = 0 max_w13_scale_fp8 = max_w13_scales[expert_id] - for shard_id in range(2): + for shard_id in range(self.moe.w13_num_shards): if layer.w13_weight_scale[expert_id][shard_id] != max_w13_scale_fp8: int4_rescale = ( layer.w13_weight_scale[expert_id][shard_id] / max_w13_scale_fp8 @@ -1193,7 +1197,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): w13_weight = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, self.get_packed_dim(hidden_size, self.weight_dtype), dtype=params_dtype, ), @@ -1220,7 +1224,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.ones( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // OCP_MX_BLOCK_SIZE, dtype=params_dtype, ), @@ -1245,7 +1249,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): w13_bias = torch.nn.Parameter( torch.zeros( num_experts, - 2 * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, dtype=torch.float32, ), requires_grad=False, @@ -1499,13 +1503,12 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): layer.quant_config = self.quant_config weight_dtype = torch.uint8 weight_scale_dtype = torch.float8_e4m3fn - w13_num_shards = 2 if self.moe.is_act_and_mul else 1 # GEMM 1 - w13 weight w13_weight = torch.nn.Parameter( torch.empty( num_experts, - w13_num_shards * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, # 2 fp4 items are packed in the input dimension hidden_size // 2, dtype=weight_dtype, @@ -1533,7 +1536,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): w13_weight_scale = torch.nn.Parameter( torch.empty( num_experts, - w13_num_shards * intermediate_size_per_partition, + self.moe.w13_num_shards * intermediate_size_per_partition, hidden_size // self.group_size, dtype=weight_scale_dtype, ), @@ -1563,7 +1566,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): ) w13_weight_scale_2 = torch.nn.Parameter( - torch.empty(num_experts, w13_num_shards, dtype=torch.float32), + torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32), requires_grad=False, ) layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2) @@ -1578,7 +1581,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod): # Input global scales (per-tensor FP32 scales) w13_input_scale_2 = torch.nn.Parameter( - torch.empty(num_experts, w13_num_shards, dtype=torch.float32), + torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32), requires_grad=False, ) layer.register_parameter("w13_input_scale_2", w13_input_scale_2)