[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)
Signed-off-by: R <Ganesh.R@amd.com> Signed-off-by: Ganesh R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
4d6874d137
commit
e12291d733
+66
-2
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w2_weight_packed", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
# Per-expert biases, as gpt-oss carries.
|
||||
if self.moe.has_bias:
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
if self.strategy == "channel":
|
||||
num_groups_w2 = num_groups_w13 = 1
|
||||
self.group_size = -1
|
||||
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
|
||||
# Monolithic experts route inside apply(), so like the unquantized
|
||||
# CPU path they need the router config that only the layer carries.
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
swigluoai_w13_interleave_perm,
|
||||
)
|
||||
|
||||
# Reorder before the conversion below, which repacks w13 for the kernel.
|
||||
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
|
||||
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
|
||||
perm = swigluoai_w13_interleave_perm(
|
||||
self.experts_cls,
|
||||
self.moe.activation,
|
||||
layer.w13_weight_packed.size(1),
|
||||
layer.w13_weight_packed.device,
|
||||
)
|
||||
if perm is not None:
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w13_weight_packed",
|
||||
layer.w13_weight_packed.data[:, perm].contiguous(),
|
||||
)
|
||||
replace_parameter(
|
||||
layer,
|
||||
"w13_weight_scale",
|
||||
layer.w13_weight_scale.data[:, perm].contiguous(),
|
||||
)
|
||||
if self.moe.has_bias:
|
||||
replace_parameter(
|
||||
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
|
||||
)
|
||||
|
||||
# Process weights using the shared oracle infrastructure
|
||||
converted = convert_to_wna16_moe_kernel_format(
|
||||
backend=self.wna16_backend,
|
||||
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
|
||||
w13_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
|
||||
if converted is None:
|
||||
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
w2_input_global_scale,
|
||||
_, # w13_bias
|
||||
_, # w2_bias
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = converted
|
||||
|
||||
# Replace common parameters
|
||||
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
if w13_bias is not None:
|
||||
replace_parameter(layer, "w13_bias", w13_bias)
|
||||
if w2_bias is not None:
|
||||
replace_parameter(layer, "w2_bias", w2_bias)
|
||||
|
||||
# CPU fused_experts_cpu requires zero points even for symmetric quant.
|
||||
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
|
||||
if (
|
||||
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
|
||||
Reference in New Issue
Block a user