From 3b1ef03be4a3c7f11b801ec3f2aa806b8c37573b Mon Sep 17 00:00:00 2001 From: PikaPikachu Date: Wed, 13 May 2026 19:59:49 +0800 Subject: [PATCH] [Bugfix][Quark] Fix W8A8 INT8 garbage outputs on Step-3.5-Flash (and other 3-key fused-MoE Quark exports) (#41892) Signed-off-by: kangletian --- .../layers/quantization/quark/quark_moe.py | 25 +++++++++++++++++++ .../quark/schemes/quark_w8a8_int8.py | 19 +++++++++++--- vllm/model_executor/models/step3p5.py | 6 +++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 100192de83..ca881b4ff8 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEParallelConfig, FusedMoEQuantConfig, fp8_w8a8_moe_quant_config, + int8_w8a8_moe_quant_config, mxfp4_w4a8_moe_quant_config, mxfp4_w4a16_moe_quant_config, ocp_mx_moe_quant_config, @@ -708,6 +709,20 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): layer.w2_input_scale.max(), requires_grad=False ) + # Per-channel scales: 2D [E, N] -> 3D [E, N, 1] for the int8 MoE kernel. + if self.weight_qscheme == "per_channel": + for attr in ("w13_weight_scale", "w2_weight_scale"): + param = getattr(layer, attr, None) + if param is not None and param.dim() == 2: + replace_parameter( + layer, + attr, + torch.nn.Parameter( + param.data.unsqueeze(-1).contiguous(), + requires_grad=False, + ), + ) + # For per-tensor weights, merge w1/w3 scales into single per-expert if self.weight_qscheme == "per_tensor": assert layer.w13_weight_scale is not None @@ -736,6 +751,16 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod): def get_fused_moe_quant_config( self, layer: torch.nn.Module ) -> FusedMoEQuantConfig | None: + if self.weight_qscheme == "per_channel" and not self.static_input_scales: + return int8_w8a8_moe_quant_config( + w1_scale=layer.w13_weight_scale, + w2_scale=layer.w2_weight_scale, + a1_scale=layer.w13_input_scale, + a2_scale=layer.w2_input_scale, + w1_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), + per_act_token_quant=True, + ) is_dynamic = not self.static_input_scales is_per_channel = self.weight_qscheme == "per_channel" return FusedMoEQuantConfig.make( diff --git a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py index 2afbe521c4..1f0c5cbffd 100644 --- a/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py @@ -47,6 +47,17 @@ class QuarkW8A8Int8(QuarkScheme): ): layer.logical_widths = output_partition_sizes + # Quark stores per-channel weight_scale as 1D [N]; reshape to [N, 1]. + def _scale_weight_loader( + param: torch.nn.Parameter, + loaded_weight: torch.Tensor, + *args, + **kwargs, + ): + if loaded_weight.dim() == 1: + loaded_weight = loaded_weight.unsqueeze(-1) + return weight_loader(param, loaded_weight, *args, **kwargs) + self.kernel = init_int8_linear_kernel( is_channelwise=(self.qscheme == "per_channel"), is_static_input_scheme=(self.is_static_input_scheme is True), @@ -69,15 +80,15 @@ class QuarkW8A8Int8(QuarkScheme): # WEIGHT SCALE if self.qscheme == "per_channel": weight_scale = ChannelQuantScaleParameter( - data=torch.empty((sum(output_partition_sizes)), dtype=torch.float32), + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32), output_dim=0, - weight_loader=weight_loader, + weight_loader=_scale_weight_loader, ) ChannelQuantZPParameter = ChannelQuantScaleParameter weight_zero_point = ChannelQuantZPParameter( - data=torch.empty((sum(output_partition_sizes)), dtype=torch.int8), + data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.int8), output_dim=0, - weight_loader=weight_loader, + weight_loader=_scale_weight_loader, ) else: assert self.qscheme == "per_tensor" diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index df051fb873..cd73f6e26d 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -817,6 +817,12 @@ class Step3p5Model(nn.Module): class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts): + # Required so quantization exclude lists match fused module prefixes. + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + hf_to_vllm_mapper = WeightsMapper( orig_to_new_substr={".share_expert.": ".moe.share_expert."} )