[CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024)

Signed-off-by: R <Ganesh.R@amd.com>
Signed-off-by: Ganesh R <Ganesh.R@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ganesh R
2026-10-01 15:18:25 +08:00
committed by GitHub
co-authored by Cursor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 4d6874d137
commit e12291d733
9 changed files with 1184 additions and 15 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs.
"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs.
Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed
``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector
@@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``.
import torch
from vllm import envs
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
from .cpu import CPUWNA16LinearKernel
from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel
from .MPLinearKernel import MPLinearLayerConfig
logger = init_logger(__name__)
@@ -34,17 +35,35 @@ def _import_unpack_from_int32():
class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
"""W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``."""
"""Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or
``zentorch_woq_linear`` (W4A16)."""
@classmethod
def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]:
ok, reason = super().can_implement(c)
if not ok:
return ok, reason
if not current_platform.is_zen_cpu():
return False, "ZentorchWNA16 requires an AMD Zen CPU."
# Not super(): its N/K multiple-of-32 rule is a oneDNN packing
# constraint the zentorch repack does not share.
if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES:
return (
False,
f"Quant type ({c.weight_type}) not supported by "
"CPUWNA16, supported types are: "
f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}",
)
# -1 is per-channel, one group spanning K. Anything else has to divide
# K, which is stricter than the parent's multiple-of-2 rule.
if c.group_size != -1:
in_features = c.partition_weight_shape[0]
if c.group_size <= 0 or in_features % c.group_size != 0:
return (
False,
f"Group size ({c.group_size}) must divide input size "
f"({in_features})",
)
if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]):
return (
False,
@@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
num_groups = weight_scale.shape[1]
return num_groups > 0 and in_features % num_groups == 0
def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool:
"""Repack ``layer`` into the DA8W4 layout, or leave it untouched.
Returns False without mutating the layer when it is not eligible, so
the caller falls through to the W4A16 paths.
"""
if not envs.VLLM_CPU_INT4_W4A8:
return False
if not has_zentorch_op(["zentorch_dynamic_qlinear"]):
return False
# DA8W4 is symmetric-only, and the kernel rejects f32 activations.
if self.config.zero_points or self.config.weight_type == scalar_types.uint4:
return False
if self.config.act_type != torch.bfloat16:
return False
if not self._zentorch_woq_eligible(layer):
return False
weight_q = getattr(layer, self.w_q_name)
weight_s = getattr(layer, self.w_s_name)
weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q
weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s
bits = self.config.weight_type.mantissa
pack_factor = torch.iinfo(weight_packed.dtype).bits // bits
out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1]
in_features = weight_packed.shape[1] * pack_factor
# AOCL sym_quant requires K/G to be a multiple of 4; K must be even to
# pack two s4 values per byte.
if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0:
return False
if self.w_zp_name is not None:
setattr(layer, self.w_zp_name, None)
unpack_from_int32 = _import_unpack_from_int32()
weight_unpacked = unpack_from_int32(
weight_packed,
bits,
torch.Size([out_features, in_features]),
packed_dim=weight_q.packed_dim,
)
# The WOQ repack packs 8 int4 per int32, which on little-endian is the
# same byte stream as s4 [N, K/2]; the kernel takes either view.
layer._zentorch_da8w4_packed = (
torch.ops.zentorch.zentorch_woq_repack_weight.default(
weight_unpacked.to(torch.int8).contiguous()
).view(torch.int8)
)
# CT stores scales as [N, G]; DA8W4 wants per-group {G, N}.
layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous()
for param_name in (self.w_q_name, self.w_s_name):
param = getattr(layer, param_name, None)
if param is not None and hasattr(param, "data"):
param.data = torch.empty(0)
layer._zentorch_kind = "compressed_tensors_w4a8_da8w4"
layer._zentorch_da8w4 = True
layer._zentorch_processed_weights = True
logger.info_once(
"[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) "
"(weight_type=%s, group_size=%d)",
self.config.weight_type,
in_features // num_groups,
)
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Repack CT GPTQ weights into the zentorch WOQ layout.
"""Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout.
Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading``
via ``super()`` when the layer doesn't satisfy
@@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
if getattr(layer, "_zentorch_processed_weights", False):
return
if self._maybe_process_da8w4_weights(layer):
return
if not self._zentorch_woq_eligible(layer):
logger.info_once(
"[zen_cpu] ZentorchWNA16 fast path not eligible for this "
@@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel):
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
if getattr(layer, "_zentorch_da8w4", False):
# The kernel reads bias through a raw pointer, so it must be
# contiguous; the weight and scales already are.
return torch.ops.zentorch.zentorch_dynamic_qlinear.default(
x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16),
layer._zentorch_da8w4_packed,
layer._zentorch_da8w4_scale,
bias.contiguous() if bias is not None else None,
)
if getattr(layer, "_zentorch_processed_weights", False):
return torch.ops.zentorch.zentorch_woq_linear.default(
x,
@@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Weight layout
# ===========================================================================
def swigluoai_w13_interleave_perm(
experts_cls: type[mk.FusedMoEExperts],
activation: MoEActivation,
two_i: int,
device: torch.device,
) -> torch.Tensor | None:
"""Index taking a half-split w13 ``[gate | up]`` to the interleaved order
ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed.
Callers apply it along whichever axis holds their output channels."""
if not getattr(experts_cls, "requires_interleaved_w13", False):
return None
if activation != MoEActivation.SWIGLUOAI:
return None
i = two_i // 2
return torch.stack(
[torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)],
dim=1,
).flatten()
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Zentorch fused MoE experts for AMD Zen CPUs."""
from collections.abc import Callable
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
_moe_activation_to_str,
has_zentorch_op,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kInt4Static,
kInt4Static32,
)
class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""DA8W4 (W4A8) group-quantized monolithic MoE experts."""
# swiglu_oai_mul reads gate/up interleaved, not in the half-split order the
# weight loader leaves behind.
requires_interleaved_w13 = True
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Defaults hold until process_weights_after_loading sees the layer.
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
)
self.custom_routing_function: Callable | None = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Capture the router config that the monolithic apply() signature
cannot carry; Custom routing has no other source for it."""
self.renormalize = layer.renormalize
self.custom_routing_function = layer.custom_routing_function
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"])
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return supported, reason
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
# The grouped GEMM needs tokens spread over at least two experts, which
# top_k=1 cannot satisfy once decode drops to a single token.
if moe_config.experts_per_token < 2:
return False, "kernel requires experts_per_token >= 2"
return True, None
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# apply() passes select_experts' global ids straight to the kernel, so
# every expert has to be resident on the rank.
return not moe_parallel_config.use_ep
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) in [
(kInt4Static, None),
(kInt4Static32, None),
]
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.Custom,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"ZentorchExpertsInt4 does not support "
"apply_router_weight_on_input=True."
)
if (
self.moe_config.routing_method == RoutingMethodType.Custom
and self.custom_routing_function is None
):
raise RuntimeError(
"ZentorchExpertsInt4 needs the model's custom_routing_function "
"for RoutingMethodType.Custom."
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
w1,
w2,
self.w1_bias,
self.w2_bias,
topk_weights,
topk_ids,
False, # skip_weighted
_moe_activation_to_str(activation),
self.w1_scale,
self.w2_scale,
)
return output
__all__ = ["ZentorchExpertsInt4"]
@@ -12,6 +12,7 @@ from compressed_tensors.quantization import (
import vllm._custom_ops as ops
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config.kernel import MoEBackend
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.config import (
@@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum):
BATCHED_MARLIN = "BATCHED_MARLIN"
HUMMING = "HUMMING"
CPU = "CPU"
ZEN_CPU = "ZEN_CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
TRITON = "TRITON"
XPU = "XPU"
@@ -98,6 +100,12 @@ def backend_to_kernel_cls(
)
return [CPUExpertsInt4]
elif backend == WNA16MoEBackend.ZEN_CPU:
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return [ZentorchExpertsInt4]
elif backend == WNA16MoEBackend.EMULATION:
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
Int4EmulationTritonExperts,
@@ -117,7 +125,7 @@ def backend_to_kernel_cls(
def _get_priority_backends() -> list[WNA16MoEBackend]:
"""Get available backends in priority order based on platform and config."""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -160,6 +168,27 @@ def _backend_incompatibility_reason(
from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
if backend == WNA16MoEBackend.ZEN_CPU:
if not envs.VLLM_CPU_INT4_W4A8:
return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path"
if may_have_zp:
return "zero points are not supported"
if isinstance(quant_config, MoeWNA16Config):
return "the MoeWNA16 weight layout is not supported"
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
return "GPTQ activation ordering is not supported"
if (
isinstance(quant_config, QuantizationArgs)
and quant_config.actorder == "group"
):
return "group activation ordering is not supported"
group_size = getattr(quant_config, "group_size", None)
if group_size is None or group_size <= 0:
return "DA8W4 requires group-quantized weights"
# AOCL sym_quant requires the group size to be a multiple of 4.
if group_size % 4 != 0:
return f"group size {group_size} is not a multiple of 4"
if backend == WNA16MoEBackend.TRITON:
if may_have_bias:
return "expert bias is not supported"
@@ -403,10 +432,14 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
# grouped/indexed experts, and Int4EmulationTritonExperts
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4,
# ZentorchExpertsInt4, the Humming grouped/indexed experts, and
# Int4EmulationTritonExperts
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
MarlinExperts,
BatchedMarlinExperts,
@@ -414,6 +447,7 @@ def make_wna16_moe_kernel(
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
ZentorchExpertsInt4,
Int4EmulationTritonExperts,
Rdna3WNA16Experts,
)
@@ -1034,6 +1068,64 @@ def _process_weights_cpu(
)
def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``."""
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
_import_unpack_from_int32,
)
num_experts, out_features, _ = packed.shape
# packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1.
unpacked = _import_unpack_from_int32()(
packed,
4,
torch.Size([num_experts, out_features, in_features]),
packed_dim=1,
)
# CT is already N-first, which is what the repack consumes.
repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default
return torch.stack([repack_op(w) for w in unpacked])
def _process_weights_zen_cpu(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""Zen CPU INT4 DA8W4 (W4A8) weight post-processing."""
# CT WNA16 buffers are N-first, so the scales are [E, N, G].
hidden_size = w2_scale.shape[1]
intermediate_size = w13_scale.shape[1] // 2
# The kernel reads per-group scales as [E, G, N], so N-first needs a swap.
return (
_zen_repack_s4(w13.data, hidden_size),
_zen_repack_s4(w2.data, intermediate_size),
w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(),
None, # w13_qzeros (symmetric)
None, # w2_qzeros (symmetric)
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias,
w2_bias,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.ZEN_CPU:
return _process_weights_zen_cpu(
w13,
w2,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
layer.register_parameter("w2_weight_packed", w2_weight)
set_weight_attrs(w2_weight, extra_weight_attrs)
# Per-expert biases, as gpt-oss carries.
if self.moe.has_bias:
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
w13_bias = torch.nn.Parameter(
torch.zeros(
num_experts,
w13_num_shards * intermediate_size_per_partition,
dtype=params_dtype,
),
requires_grad=False,
)
layer.register_parameter("w13_bias", w13_bias)
set_weight_attrs(w13_bias, extra_weight_attrs)
w2_bias = torch.nn.Parameter(
torch.zeros(num_experts, hidden_size, dtype=params_dtype),
requires_grad=False,
)
layer.register_parameter("w2_bias", w2_bias)
set_weight_attrs(w2_bias, extra_weight_attrs)
if self.strategy == "channel":
num_groups_w2 = num_groups_w13 = 1
self.group_size = -1
@@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
routing_tables=layer._expert_routing_tables(),
)
if self.wna16_backend == WNA16MoEBackend.ZEN_CPU:
# Monolithic experts route inside apply(), so like the unquantized
# CPU path they need the router config that only the layer carries.
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
# Reorder before the conversion below, which repacks w13 for the kernel.
# The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight
# [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias.
perm = swigluoai_w13_interleave_perm(
self.experts_cls,
self.moe.activation,
layer.w13_weight_packed.size(1),
layer.w13_weight_packed.device,
)
if perm is not None:
replace_parameter(
layer,
"w13_weight_packed",
layer.w13_weight_packed.data[:, perm].contiguous(),
)
replace_parameter(
layer,
"w13_weight_scale",
layer.w13_weight_scale.data[:, perm].contiguous(),
)
if self.moe.has_bias:
replace_parameter(
layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous()
)
# Process weights using the shared oracle infrastructure
converted = convert_to_wna16_moe_kernel_format(
backend=self.wna16_backend,
@@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_scale=layer.w2_weight_scale,
w13_qzeros=getattr(layer, "w13_weight_zero_point", None),
w2_qzeros=getattr(layer, "w2_weight_zero_point", None),
w13_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
if converted is None:
@@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
_, # w13_bias
_, # w2_bias
w13_bias,
w2_bias,
) = converted
# Replace common parameters
@@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if w13_bias is not None:
replace_parameter(layer, "w13_bias", w13_bias)
if w2_bias is not None:
replace_parameter(layer, "w2_bias", w2_bias)
# CPU fused_experts_cpu requires zero points even for symmetric quant.
# EMULATION bakes ZP into the dequantized bf16 weights — ZP is None.
if (
@@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
num_bits=self.num_bits,
w1_zp=getattr(layer, "w13_weight_zero_point", None),
w2_zp=getattr(layer, "w2_weight_zero_point", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
+4 -1
View File
@@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
"w2_bias": "gpt_oss_w2",
"w13_weight_scale": "gpt_oss_w13",
"w2_weight_scale": "gpt_oss_w2",
"w13_weight_packed": "gpt_oss_w13",
"w2_weight_packed": "gpt_oss_w2",
}
@@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts):
) -> None:
tp_rank = self.moe_config.moe_parallel_config.tp_rank
is_w13 = shard_id == "gpt_oss_w13"
is_weight = weight_name.endswith("_weight")
is_weight = weight_name.endswith(("_weight", "_weight_packed"))
is_partitioned_scale = weight_name.endswith("_weight_scale")
if is_weight:
@@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts):
) or quant_method_name in (
"CompressedTensorsW4A4Nvfp4MoEMethod",
"Nvfp4OnlineMoEMethod",
"CompressedTensorsWNA16MoEMethod",
):
self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id)
else: