[CPU] Support CPU W4A16 INT4 MoE (#43409)

Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
Yuwen Zhou
2026-06-12 07:12:37 +00:00
committed by GitHub
parent a2c72d4388
commit 0cd9b7af25
8 changed files with 685 additions and 22 deletions
+1 -1
View File
@@ -91,7 +91,7 @@ steps:
- tests/quantization/test_cpu_wna16.py
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
pytest -x -v -s tests/quantization/test_cpu_wna16.py"
@@ -496,5 +496,258 @@ def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
# INT4 W4A16 group-quantized MoE
def _pack_int4_gptq(w_int4: torch.Tensor) -> torch.Tensor:
"""Pack INT4 values [N, K] → [N, K//8] int32 along K dim (GPTQ format)."""
N, K = w_int4.shape
assert K % 8 == 0
w = w_int4.to(torch.int32)
w_packed = torch.zeros(N, K // 8, dtype=torch.int32)
for j in range(8):
w_packed |= (w[:, j::8] & 0xF) << (j * 4)
return w_packed
def _pack_int4_awq(w_int4: torch.Tensor) -> torch.Tensor:
"""Pack INT4 values [..., N] → [..., N//8] int32 along last dim (AWQ format)."""
# AWQ packing bitshifts: indices {0,4,1,5,2,6,3,7} * 4 bits each
_AWQ_BITSHIFTS = [0, 16, 4, 20, 8, 24, 12, 28]
N = w_int4.shape[-1]
assert N % 8 == 0
w = w_int4.to(torch.int32)
w_packed = torch.zeros(*w.shape[:-1], N // 8, dtype=torch.int32)
for j, shift in enumerate(_AWQ_BITSHIFTS):
w_packed |= (w[..., j::8] & 0xF) << shift
return w_packed
def _ref_int4_moe(
a: torch.Tensor,
w1_int4: torch.Tensor,
w2_int4: torch.Tensor,
w1_zeros: torch.Tensor | None,
w2_zeros: torch.Tensor | None,
w1_s: torch.Tensor,
w2_s: torch.Tensor,
topk_weight: torch.Tensor,
topk_ids: torch.Tensor,
group_size: int,
) -> torch.Tensor:
"""Reference INT4 W4A16 group-quantized fused MoE in pure torch."""
B = a.shape[0]
topk = topk_ids.size(1)
K_out = a.shape[1]
out = torch.zeros(B, topk, K_out, dtype=torch.float32)
for b in range(B):
for t in range(topk):
eid = topk_ids[b, t].item()
x = a[b : b + 1].float()
# Dequantize w1: [K, 2*N], groups along K (input dim)
K_dim = w1_int4.shape[1]
w1_dq = torch.zeros(K_dim, w1_int4.shape[2], dtype=torch.float32)
for g in range(w1_s.shape[1]):
k_start = g * group_size
k_end = min((g + 1) * group_size, K_dim)
zp = w1_zeros[eid, g, :].float() if w1_zeros is not None else 8.0
w1_dq[k_start:k_end, :] = (
w1_int4[eid, k_start:k_end, :].float() - zp
) * w1_s[eid, g, :].float()
ic = torch.matmul(x, w1_dq) # [1, K] @ [K, 2*N] → [1, 2*N]
ic = _silu_and_mul(ic) # [1, N]
# Dequantize w2: [N, K], groups along N (input dim)
N_dim = w2_int4.shape[1]
w2_dq = torch.zeros(N_dim, w2_int4.shape[2], dtype=torch.float32)
for g in range(w2_s.shape[1]):
n_start = g * group_size
n_end = min((g + 1) * group_size, N_dim)
zp = w2_zeros[eid, g, :].float() if w2_zeros is not None else 8.0
w2_dq[n_start:n_end, :] = (
w2_int4[eid, n_start:n_end, :].float() - zp
) * w2_s[eid, g, :].float()
oc = torch.matmul(ic, w2_dq) # [1, N] @ [N, K] → [1, K]
out[b, t] = oc.squeeze(0)
return (out * topk_weight.unsqueeze(-1)).sum(dim=1).to(a.dtype)
def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
"""Create INT4 MoE weights in GPTQ or AWQ packed format.
Canonical layout (input × output):
w1_int4: [E, K, 2*N] w2_int4: [E, N, K]
GPTQ packed (pack transposed weight along input/K dim):
w1_packed: [E, K//8, 2*N] w2_packed: [E, N//8, K]
zeros: actual int4 zero points, same packing as weights
AWQ packed (pack along output/N dim):
w1_packed: [E, K, 2*N//8] w2_packed: [E, N, K//8]
zeros: actual int4 zero points, same packing as weights
Returns:
w1_int4, w2_int4,
w1_packed, w2_packed,
w1_zeros, w2_zeros,
w1_zeros_packed, w2_zeros_packed,
w1_s, w2_s
"""
w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)
num_groups_w1 = K // group_size
num_groups_w2 = N // group_size
w1_s = (
torch.randn(E, num_groups_w1, 2 * N, dtype=torch.bfloat16) * 0.01
).abs() + 0.001
w2_s = (torch.randn(E, num_groups_w2, K, dtype=torch.bfloat16) * 0.01).abs() + 0.001
if quant_algo == ops.CPUQuantAlgo.GPTQ:
# Pack: canonical [E, K, 2*N] → transpose [E, 2*N, K] → GPTQ pack
# [E, 2*N, K//8] → transpose [E, K//8, 2*N]
w1_t = w1_int4.transpose(1, 2).contiguous() # [E, 2*N, K]
w1_packed = (
torch.stack([_pack_int4_gptq(w1_t[e]) for e in range(E)])
.transpose(1, 2)
.contiguous()
) # [E, K//8, 2*N]
w2_t = w2_int4.transpose(1, 2).contiguous() # [E, K, N]
w2_packed = (
torch.stack([_pack_int4_gptq(w2_t[e]) for e in range(E)])
.transpose(1, 2)
.contiguous()
) # [E, N//8, K]
w1_zeros = w2_zeros = None
w1_zeros_packed = torch.full(
(E, num_groups_w1, 2 * N // 8), 0x77777777, dtype=torch.int32
)
w2_zeros_packed = torch.full(
(E, num_groups_w2, K // 8), 0x77777777, dtype=torch.int32
)
else: # AWQ
# Asymmetric: actual zero points, packed along output dim.
w1_zeros = torch.randint(1, 15, (E, num_groups_w1, 2 * N), dtype=torch.int32)
w2_zeros = torch.randint(1, 15, (E, num_groups_w2, K), dtype=torch.int32)
w1_packed = torch.stack(
[_pack_int4_awq(w1_int4[e]) for e in range(E)]
) # [E, K, 2*N//8]
w2_packed = torch.stack(
[_pack_int4_awq(w2_int4[e]) for e in range(E)]
) # [E, N, K//8]
w1_zeros_packed = torch.stack(
[_pack_int4_awq(w1_zeros[e]) for e in range(E)]
) # [E, K//gs, 2*N//8]
w2_zeros_packed = torch.stack(
[_pack_int4_awq(w2_zeros[e]) for e in range(E)]
) # [E, N//gs, K//8]
return (
w1_int4,
w2_int4,
w1_packed,
w2_packed,
w1_zeros,
w2_zeros,
w1_zeros_packed,
w2_zeros_packed,
w1_s,
w2_s,
)
INT4_MOE_CONFIGS = [
# (N, K, E, topk, group_size)
(256, 512, 8, 2, 128),
(512, 256, 8, 2, 128),
(512, 512, 8, 4, 128),
(768, 2048, 8, 2, 128),
]
@pytest.mark.parametrize("M", [1, 2, 64, 121])
@pytest.mark.parametrize("N,K,E,topk,group_size", INT4_MOE_CONFIGS)
@pytest.mark.parametrize("quant_algo", [ops.CPUQuantAlgo.GPTQ, ops.CPUQuantAlgo.AWQ])
@pytest.mark.parametrize("seed", [0])
def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed):
"""Test fused_experts_cpu INT4 W4A16 for both GPTQ and AWQ quant formats."""
set_random_seed(seed)
a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
(
w1_int4,
w2_int4,
w1_packed,
w2_packed,
w1_zeros,
w2_zeros,
w1_zeros_packed,
w2_zeros_packed,
w1_s,
w2_s,
) = _make_int4_moe_weights(E, N, K, group_size, quant_algo)
score = torch.randn(M, E, dtype=torch.bfloat16)
score = torch.softmax(score, dim=-1, dtype=torch.float32)
topk_weight, topk_ids = torch.topk(score, topk)
topk_ids = topk_ids.to(torch.int32)
ref_out = _ref_int4_moe(
a,
w1_int4,
w2_int4,
w1_zeros,
w2_zeros,
w1_s,
w2_s,
topk_weight,
topk_ids,
group_size,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int4_moe_layer_for_cpu,
)
(blocked_w1, blocked_w2, blocked_s1, blocked_s2, blocked_z1, blocked_z2) = (
prepare_int4_moe_layer_for_cpu(
w1_packed,
w2_packed,
w1_s,
w2_s,
quant_algo=quant_algo,
w13_zeros=w1_zeros_packed,
w2_zeros=w2_zeros_packed,
)
)
out = ops.fused_experts_cpu(
a.clone(),
blocked_w1,
blocked_w2,
topk_weight,
topk_ids,
False, # inplace
ops.CPUQuantMethod.INT4_W4A8,
blocked_s1,
blocked_s2,
blocked_z1,
blocked_z2,
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))
+3
View File
@@ -16,6 +16,9 @@ MODELS = [
"Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear
"Qwen/Qwen3-30B-A3B-FP8", # FP8 W8A16 block-quantized MoE
"openai/gpt-oss-20b", # MXFP4 W4A16
"QuixiAI/Qwen3-30B-A3B-AWQ", # AWQ W4A16 MoE
"Qwen/Qwen3-30B-A3B-GPTQ-Int4", # GPTQ W4A16 MoE
"RedHatAI/Qwen3-30B-A3B-quantized.w4a16", # compressed-tensors W4A16 MoE
]
DTYPE = ["bfloat16"]
@@ -5,7 +5,12 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._custom_ops import CPUQuantMethod, fused_experts_cpu
from vllm._custom_ops import (
CPUQuantAlgo,
CPUQuantMethod,
convert_weight_packed_scale_zp,
fused_experts_cpu,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -17,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
kFp8Static128BlockSym,
kInt4Static,
kMxfp4Static,
)
from vllm.platforms import current_platform
@@ -318,3 +324,209 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
limit,
True, # is_vnni
)
def prepare_int4_moe_layer_for_cpu(
w13_packed: torch.Tensor,
w2_packed: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
quant_algo: CPUQuantAlgo = CPUQuantAlgo.GPTQ,
w13_zeros: torch.Tensor | None = None,
w2_zeros: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Repack INT4 MoE weights via convert_weight_packed_scale_zp for CPU.
Args:
w13_packed: [E, K//8, 2*I] int32 (packed int4)
w2_packed: [E, I//8, K] int32 (packed int4)
w13_scale: [E, num_groups, 2*I] float16/bf16
w2_scale: [E, num_groups, K] float16/bf16
quant_algo: CPUQuantAlgo.GPTQ or CPUQuantAlgo.AWQ
w13_zeros: optional [E, num_groups, N//8] int32 packed zeros.
If None, synthetic zeros are created for symmetric quant.
w2_zeros: optional [E, num_groups, N//8] int32 packed zeros.
If None, synthetic zeros are created for symmetric quant.
Returns:
(blocked_w13, blocked_w2, blocked_s13, blocked_s2,
blocked_z13, blocked_z2)
"""
E = w13_packed.size(0)
# No qzeros are available in compressed-tensors symmetric checkpoints.
# The GPTQ unpack kernel (unpack_4bit_to_32bit_signed) adds +1 to stored zeros,
# so we store 7 per nibble: 0x77777777 → +1 → 8.
if w13_zeros is None:
num_groups_w13 = w13_scale.size(1)
N_w13 = w13_scale.size(2) # 2*I
_zp = 0x77777777
w13_zeros = torch.full(
(E, num_groups_w13, N_w13 // 8),
_zp,
dtype=torch.int32,
)
if w2_zeros is None:
num_groups_w2 = w2_scale.size(1)
N_w2 = w2_scale.size(2) # K
_zp = 0x77777777
w2_zeros = torch.full(
(E, num_groups_w2, N_w2 // 8),
_zp,
dtype=torch.int32,
)
blocked_w13, blocked_z13, blocked_s13 = convert_weight_packed_scale_zp(
w13_packed, w13_zeros, w13_scale, quant_algo
)
blocked_w2, blocked_z2, blocked_s2 = convert_weight_packed_scale_zp(
w2_packed, w2_zeros, w2_scale, quant_algo
)
return (blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""CPU INT4 W4A16 group-quantized monolithic MoE experts.
Weights are int4 (packed), activations are bf16/fp16.
Internally uses int8 compute via fused_experts_cpu with INT4_W4A8.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(
moe_config,
quant_config,
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cpu()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return True
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
SUPPORTED_W_A = [
(kInt4Static, None),
]
return (weight_key, activation_key) in SUPPORTED_W_A
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
def supports_expert_map(self) -> bool:
return False
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
"CPUExpertsInt4 (W4A16) does not support "
"apply_router_weight_on_input=True. "
)
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
select_experts,
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.INT4_W4A8,
self.w1_scale,
self.w2_scale,
self.w1_zp,
self.w2_zp,
None, # block_size
None, # w1_bias
None, # w2_bias
None, # alpha
None, # limit
True, # is_vnni
)
@@ -45,6 +45,7 @@ logger = init_logger(__name__)
class WNA16MoEBackend(Enum):
MARLIN = "MARLIN"
BATCHED_MARLIN = "BATCHED_MARLIN"
CPU = "CPU"
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
XPU = "XPU"
@@ -65,6 +66,12 @@ def backend_to_kernel_cls(
)
return [XPUExpertsWNA16]
elif backend == WNA16MoEBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt4,
)
return [CPUExpertsInt4]
else:
raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}")
@@ -73,6 +80,8 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
"""
Get available backends in priority order based on platform and config.
"""
if current_platform.is_cpu():
return [WNA16MoEBackend.CPU]
if current_platform.is_xpu():
return [WNA16MoEBackend.XPU]
@@ -210,17 +219,21 @@ def make_wna16_moe_kernel(
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUExpertsInt4,
)
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsWNA16,
)
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts
# and BatchedMarlinExperts
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
# BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
assert experts_cls in (
MarlinExperts,
BatchedMarlinExperts,
TrtLlmMxint4ExpertsMonolithic,
XPUExpertsWNA16,
CPUExpertsInt4,
)
is_monolithic = experts_cls.is_monolithic()
@@ -683,6 +696,117 @@ def _process_awq_weights_marlin(
)
def _process_weights_cpu(
quant_config: QuantizationConfig | QuantizationArgs | None,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
w13_g_idx: torch.Tensor | None = None,
w2_g_idx: torch.Tensor | None = None,
w13_qzeros: torch.Tensor | None = None,
w2_qzeros: torch.Tensor | None = None,
w13_bias: torch.Tensor | None = None,
w2_bias: torch.Tensor | None = None,
) -> tuple[
torch.Tensor, # w13_qweight
torch.Tensor, # w2_qweight
torch.Tensor, # w13_scales
torch.Tensor, # w2_scales
torch.Tensor | None, # w13_g_idx
torch.Tensor | None, # w2_g_idx
torch.Tensor | None, # w13_g_idx_sort_indices
torch.Tensor | None, # w2_g_idx_sort_indices
torch.Tensor | None, # w13_qzeros
torch.Tensor | None, # w2_qzeros
torch.Tensor | None, # w13_input_global_scale
torch.Tensor | None, # w2_input_global_scale
torch.Tensor | None, # w13_bias
torch.Tensor | None, # w2_bias
]:
"""CPU INT4 W4A16 weight post-processing."""
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_int4_moe_layer_for_cpu,
)
from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQConfig,
)
from vllm.model_executor.layers.quantization.awq_marlin import (
AWQMarlinConfig,
)
# Detect packing format.
# AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim).
# GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim).
# compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim).
if isinstance(quant_config, AWQMarlinConfig):
# AWQ: K is stored unpacked in dim 1.
cpu_quant_algo = ops.CPUQuantAlgo.AWQ
elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
# GPTQ / compressed-tensors: K//8 is stored packed in dim 1.
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
raise NotImplementedError(
"CPU WNA16 MoE backend does not support GPTQ with "
"desc_act=True. The fused MoE kernel has no g_idx "
"reordering support."
)
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
else:
raise TypeError(
"CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig "
f"or QuantizationArgs, got {type(quant_config).__name__}."
)
# Determine zero points for repacking.
w13_zeros: torch.Tensor | None = None
w2_zeros: torch.Tensor | None = None
if w13_qzeros is not None:
w13_zeros = (
w13_qzeros.data.view(torch.int32)
if w13_qzeros.dtype != torch.int32
else w13_qzeros.data
)
if w2_qzeros is not None:
w2_zeros = (
w2_qzeros.data.view(torch.int32)
if w2_qzeros.dtype != torch.int32
else w2_qzeros.data
)
(
blocked_w13,
blocked_w2,
blocked_s13,
blocked_s2,
blocked_z13,
blocked_z2,
) = prepare_int4_moe_layer_for_cpu(
w13,
w2,
w13_scale,
w2_scale,
quant_algo=cpu_quant_algo,
w13_zeros=w13_zeros,
w2_zeros=w2_zeros,
)
return (
blocked_w13,
blocked_w2,
blocked_s13,
blocked_s2,
w13_g_idx,
w2_g_idx,
None, # w13_g_idx_sort_indices (unused on CPU)
None, # w2_g_idx_sort_indices (unused on CPU)
blocked_z13,
blocked_z2,
None, # w13_input_global_scale
None, # w2_input_global_scale
w13_bias.to(torch.float32) if w13_bias is not None else None,
w2_bias.to(torch.float32) if w2_bias is not None else None,
)
def _process_weights_xpu(
layer: torch.nn.Module,
quant_config: QuantizationConfig,
@@ -857,6 +981,20 @@ def convert_to_wna16_moe_kernel_format(
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.CPU:
return _process_weights_cpu(
quant_config,
w13,
w2,
w13_scale,
w2_scale,
w13_g_idx,
w2_g_idx,
w13_qzeros,
w2_qzeros,
w13_bias,
w2_bias,
)
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
return _process_weights_flashinfer(
w13,
@@ -16,6 +16,7 @@ from vllm.model_executor.kernels.linear import (
)
from vllm.model_executor.layers.fused_moe import (
FusedMoEConfig,
FusedMoEExpertsModular,
FusedMoEMethodBase,
FusedMoEQuantConfig,
FusedMoeWeightScaleSupported,
@@ -640,8 +641,11 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
device = layer.w13_qweight.device
layer.workspace = marlin_make_workspace_new(device, 4)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1
@@ -660,8 +664,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_g_idx,
w13_g_idx_sort_indices,
w2_g_idx_sort_indices,
_w13_qzeros,
_w2_qzeros,
w13_qzeros,
w2_qzeros,
w13_input_global_scale,
w2_input_global_scale,
w13_bias,
@@ -689,6 +693,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
replace_parameter(layer, "w2_g_idx", w2_g_idx)
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
if w13_qzeros is not None:
replace_parameter(layer, "w13_qzeros", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_qzeros", w2_qzeros)
if w13_input_global_scale is not None:
if hasattr(layer, "w13_input_global_scale"):
replace_parameter(
@@ -735,8 +743,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
is_k_full=self.is_k_full,
w13_g_idx=layer.w13_g_idx,
w2_g_idx=layer.w2_g_idx,
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -750,12 +758,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
w2_scale=layer.w2_scales,
weight_bits=self.quant_config.weight_bits,
group_size=self.quant_config.group_size,
w1_zp=getattr(layer, "w13_qzeros", None)
if not self.quant_config.is_sym
else None,
w2_zp=getattr(layer, "w2_qzeros", None)
if not self.quant_config.is_sym
else None,
w1_zp=getattr(layer, "w13_qzeros", None),
w2_zp=getattr(layer, "w2_qzeros", None),
w1_bias=getattr(layer, "w13_bias", None),
w2_bias=getattr(layer, "w2_bias", None),
)
@@ -794,3 +798,27 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -700,8 +700,8 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
is_k_full=self.is_k_full,
w13_g_idx=getattr(layer, "w13_g_idx", None),
w2_g_idx=getattr(layer, "w2_g_idx", None),
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
routing_tables=layer._expert_routing_tables(),
)
@@ -757,3 +757,27 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
shared_experts=shared_experts,
shared_experts_input=shared_experts_input,
)
def apply_monolithic(
self,
layer: RoutedExperts,
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
hidden_states=x,
w1=layer.w13_qweight,
w2=layer.w2_qweight,
router_logits=router_logits,
activation=layer.activation,
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
)
@@ -10,6 +10,7 @@ from compressed_tensors.quantization import (
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoEExpertsModular,
RoutedExperts,
SharedExperts,
)
@@ -414,8 +415,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
replace_parameter(layer, "w13_weight_scale", w13_scales)
replace_parameter(layer, "w2_weight_scale", w2_scales)
if not self.symmetric:
if w13_qzeros is not None:
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
if w2_qzeros is not None:
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
# Marlin-specific parameters (not needed for Flashinfer)
@@ -437,9 +439,12 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
)
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
)
if self.experts_cls is not None and issubclass(
self.experts_cls, FusedMoEExpertsModular
):
layer.workspace = marlin_make_workspace_new(
layer.w13_weight_g_idx.device, 4
)
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
layer.w13_weight = layer.w13_weight_packed