[Bugfix][Quark] Fix W8A8 INT8 garbage outputs on Step-3.5-Flash (and other 3-key fused-MoE Quark exports) (#41892)
Signed-off-by: kangletian <kangletian@hotmail.com>
This commit is contained in:
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
int8_w8a8_moe_quant_config,
|
||||
mxfp4_w4a8_moe_quant_config,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
ocp_mx_moe_quant_config,
|
||||
@@ -708,6 +709,20 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
layer.w2_input_scale.max(), requires_grad=False
|
||||
)
|
||||
|
||||
# Per-channel scales: 2D [E, N] -> 3D [E, N, 1] for the int8 MoE kernel.
|
||||
if self.weight_qscheme == "per_channel":
|
||||
for attr in ("w13_weight_scale", "w2_weight_scale"):
|
||||
param = getattr(layer, attr, None)
|
||||
if param is not None and param.dim() == 2:
|
||||
replace_parameter(
|
||||
layer,
|
||||
attr,
|
||||
torch.nn.Parameter(
|
||||
param.data.unsqueeze(-1).contiguous(),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
|
||||
# For per-tensor weights, merge w1/w3 scales into single per-expert
|
||||
if self.weight_qscheme == "per_tensor":
|
||||
assert layer.w13_weight_scale is not None
|
||||
@@ -736,6 +751,16 @@ class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
if self.weight_qscheme == "per_channel" and not self.static_input_scales:
|
||||
return int8_w8a8_moe_quant_config(
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
per_act_token_quant=True,
|
||||
)
|
||||
is_dynamic = not self.static_input_scales
|
||||
is_per_channel = self.weight_qscheme == "per_channel"
|
||||
return FusedMoEQuantConfig.make(
|
||||
|
||||
@@ -47,6 +47,17 @@ class QuarkW8A8Int8(QuarkScheme):
|
||||
):
|
||||
layer.logical_widths = output_partition_sizes
|
||||
|
||||
# Quark stores per-channel weight_scale as 1D [N]; reshape to [N, 1].
|
||||
def _scale_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
if loaded_weight.dim() == 1:
|
||||
loaded_weight = loaded_weight.unsqueeze(-1)
|
||||
return weight_loader(param, loaded_weight, *args, **kwargs)
|
||||
|
||||
self.kernel = init_int8_linear_kernel(
|
||||
is_channelwise=(self.qscheme == "per_channel"),
|
||||
is_static_input_scheme=(self.is_static_input_scheme is True),
|
||||
@@ -69,15 +80,15 @@ class QuarkW8A8Int8(QuarkScheme):
|
||||
# WEIGHT SCALE
|
||||
if self.qscheme == "per_channel":
|
||||
weight_scale = ChannelQuantScaleParameter(
|
||||
data=torch.empty((sum(output_partition_sizes)), dtype=torch.float32),
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.float32),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
weight_loader=_scale_weight_loader,
|
||||
)
|
||||
ChannelQuantZPParameter = ChannelQuantScaleParameter
|
||||
weight_zero_point = ChannelQuantZPParameter(
|
||||
data=torch.empty((sum(output_partition_sizes)), dtype=torch.int8),
|
||||
data=torch.empty((sum(output_partition_sizes), 1), dtype=torch.int8),
|
||||
output_dim=0,
|
||||
weight_loader=weight_loader,
|
||||
weight_loader=_scale_weight_loader,
|
||||
)
|
||||
else:
|
||||
assert self.qscheme == "per_tensor"
|
||||
|
||||
@@ -817,6 +817,12 @@ class Step3p5Model(nn.Module):
|
||||
|
||||
|
||||
class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
# Required so quantization exclude lists match fused module prefixes.
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={".share_expert.": ".moe.share_expert."}
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user