[feature]Add int4 quantization support for emulation moe backend (#48451)
Signed-off-by: Qiang Li <qiang.li2@amd.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -887,6 +887,9 @@ def int4_w4a16_moe_quant_config(
|
||||
block_shape: list[int] | None = None,
|
||||
a1_gscale: torch.Tensor | None = None,
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and int4 weights.
|
||||
@@ -897,6 +900,9 @@ def int4_w4a16_moe_quant_config(
|
||||
_a2=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a2_gscale),
|
||||
_w1=FusedMoEQuantDesc("int4", group_shape, w1_scale, None, w1_zp, w1_bias),
|
||||
_w2=FusedMoEQuantDesc("int4", group_shape, w2_scale, None, w2_zp, w2_bias),
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
)
|
||||
|
||||
|
||||
@@ -950,6 +956,9 @@ def int8_w8a16_moe_quant_config(
|
||||
block_shape: list[int] | None = None,
|
||||
a1_gscale: torch.Tensor | None = None,
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""
|
||||
Construct a quant config for 16-bit float activations and int8 weights.
|
||||
@@ -960,6 +969,9 @@ def int8_w8a16_moe_quant_config(
|
||||
_a2=FusedMoEQuantDesc(shape=group_shape, alpha_or_gscale=a2_gscale),
|
||||
_w1=FusedMoEQuantDesc(torch.int8, group_shape, w1_scale, None, w1_zp, w1_bias),
|
||||
_w2=FusedMoEQuantDesc(torch.int8, group_shape, w2_scale, None, w2_zp, w2_bias),
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Int4 weight-only quantization emulation for MoE.
|
||||
|
||||
Weights are dequantized from packed int4 to BF16 once at load time;
|
||||
the forward pass then runs plain TritonExperts in BF16.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_moe import TritonExperts
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4Static32Asym,
|
||||
kInt4StaticAsym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Int4EmulationTritonExperts(TritonExperts):
|
||||
"""Int4 W-only MoE that dequantizes weights to BF16 at load time.
|
||||
|
||||
Weights arrive already dequantized (convert_to_wna16_moe_kernel_format
|
||||
does the unpacking); apply() simply forwards to TritonExperts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
logger.warning_once(
|
||||
"Using Int4EmulationTritonExperts MoE backend. Int4 weights are "
|
||||
"dequantized to BF16 at load time "
|
||||
)
|
||||
# Weights are dequantized to BF16 before apply() is called, so
|
||||
# TritonExperts must see them as plain float — clear the int4 dtype
|
||||
# and scales so the hidden-size assertion and kernel dispatch treat
|
||||
# them as unquantized.
|
||||
self.quant_config._w1.dtype = None
|
||||
self.quant_config._w2.dtype = None
|
||||
self.quant_config._w1.scale = None
|
||||
self.quant_config._w2.scale = None
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda_alike()
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (
|
||||
weight_key
|
||||
in (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt4StaticAsym,
|
||||
kInt4Static32Asym,
|
||||
)
|
||||
and activation_key is None
|
||||
)
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def block_shape(self) -> list[int] | None:
|
||||
return None
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
if w1.element_size() < 2:
|
||||
raise RuntimeError(
|
||||
"Int4EmulationTritonExperts.apply() received packed int4 weights "
|
||||
"(element_size < 2). Weights must be dequantized to BF16 before "
|
||||
"the forward pass via convert_to_wna16_moe_kernel_format."
|
||||
)
|
||||
return super().apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=global_num_experts,
|
||||
expert_map=expert_map,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
@@ -51,6 +51,7 @@ class WNA16MoEBackend(Enum):
|
||||
CPU = "CPU"
|
||||
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
|
||||
XPU = "XPU"
|
||||
EMULATION = "EMULATION"
|
||||
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
@@ -87,6 +88,12 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [CPUExpertsInt4]
|
||||
elif backend == WNA16MoEBackend.EMULATION:
|
||||
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
|
||||
Int4EmulationTritonExperts,
|
||||
)
|
||||
|
||||
return [Int4EmulationTritonExperts]
|
||||
else:
|
||||
raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}")
|
||||
|
||||
@@ -105,6 +112,7 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
WNA16MoEBackend.MARLIN,
|
||||
WNA16MoEBackend.BATCHED_MARLIN,
|
||||
WNA16MoEBackend.HUMMING,
|
||||
WNA16MoEBackend.EMULATION,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
|
||||
@@ -115,6 +123,7 @@ def map_wna16_backend(runner_backend: MoEBackend) -> WNA16MoEBackend:
|
||||
"marlin": WNA16MoEBackend.MARLIN,
|
||||
"humming": WNA16MoEBackend.HUMMING,
|
||||
"flashinfer_trtllm": WNA16MoEBackend.FLASHINFER_TRTLLM,
|
||||
"emulation": WNA16MoEBackend.EMULATION,
|
||||
}
|
||||
if backend := mapping.get(runner_backend):
|
||||
return backend
|
||||
@@ -215,6 +224,9 @@ def make_wna16_moe_quant_config(
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
a1_gscale: torch.Tensor | None = None,
|
||||
a2_gscale: torch.Tensor | None = None,
|
||||
gemm1_clamp_limit: float | None = None,
|
||||
gemm1_alpha: float | None = None,
|
||||
gemm1_beta: float | None = None,
|
||||
) -> FusedMoEQuantConfig:
|
||||
"""Create the FusedMoEQuantConfig for 4 or 8-bit WNA16 MoE."""
|
||||
if num_bits == 4:
|
||||
@@ -228,6 +240,9 @@ def make_wna16_moe_quant_config(
|
||||
block_shape=[0, group_size],
|
||||
a1_gscale=a1_gscale,
|
||||
a2_gscale=a2_gscale,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
)
|
||||
else:
|
||||
assert num_bits == 8
|
||||
@@ -241,6 +256,9 @@ def make_wna16_moe_quant_config(
|
||||
block_shape=[0, group_size],
|
||||
a1_gscale=a1_gscale,
|
||||
a2_gscale=a2_gscale,
|
||||
gemm1_clamp_limit=gemm1_clamp_limit,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
gemm1_beta=gemm1_beta,
|
||||
)
|
||||
|
||||
|
||||
@@ -263,19 +281,23 @@ def make_wna16_moe_kernel(
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import (
|
||||
Int4EmulationTritonExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsWNA16,
|
||||
)
|
||||
|
||||
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, and the Humming
|
||||
# grouped/indexed experts.
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming
|
||||
# grouped/indexed experts, and Int4EmulationTritonExperts
|
||||
allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = (
|
||||
MarlinExperts,
|
||||
BatchedMarlinExperts,
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
XPUExpertsWNA16,
|
||||
CPUExpertsInt4,
|
||||
Int4EmulationTritonExperts,
|
||||
)
|
||||
if backend == WNA16MoEBackend.HUMMING:
|
||||
allowed_experts += tuple(backend_to_kernel_cls(WNA16MoEBackend.HUMMING))
|
||||
@@ -1017,6 +1039,243 @@ def _humming_wna16_weight_schema(
|
||||
)
|
||||
|
||||
|
||||
def _unpack_and_dequant_int4_gptq(
|
||||
w_int32: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
qzeros: torch.Tensor | None,
|
||||
transpose_output: bool,
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.Tensor:
|
||||
"""Unpack GPTQ-packed int4 weights and dequantize to output_dtype.
|
||||
|
||||
Args:
|
||||
w_int32: packed weights, shape [E, K_packed, N] where K_packed = K//8
|
||||
(8 nibbles per int32, LSB-first in the K dimension).
|
||||
scale: per-group scales, shape [E, K//group_size, N], float16.
|
||||
qzeros: optional asymmetric zero-points, shape [E, K//gs, N//8], int32.
|
||||
None for symmetric (uint4b8 with implicit bias 8).
|
||||
transpose_output: if True return [E, N, K]; if False return [E, K, N].
|
||||
output_dtype: target floating-point dtype (bfloat16 or float16).
|
||||
|
||||
Returns:
|
||||
Dequantized weight tensor in the requested layout.
|
||||
"""
|
||||
E, K_packed, N = w_int32.shape
|
||||
K = K_packed * 8
|
||||
|
||||
# Unpack: [E, K_packed, N] -> [E, K_packed, N, 8] via bit-shifts.
|
||||
# The nibble index (last dim) enumerates K rows within each packed column,
|
||||
# so we must fuse K_packed and the nibble dim, not N and the nibble dim.
|
||||
# Permute to [E, K_packed, 8, N] before reshaping to [E, K, N].
|
||||
shifts = torch.arange(8, device=w_int32.device, dtype=torch.int32) * 4
|
||||
nibbles = (w_int32.unsqueeze(-1) >> shifts) & 0xF # [E, K_packed, N, 8]
|
||||
|
||||
# Reshape to [E, K, N]: fuse K_packed and nibble index (dim 1 and 3)
|
||||
w = nibbles.permute(0, 1, 3, 2).reshape(E, K, N).to(torch.int16)
|
||||
|
||||
if qzeros is None:
|
||||
# Symmetric uint4b8: subtract bias so the range is [-8, 7]
|
||||
w = w - 8
|
||||
else:
|
||||
# Asymmetric: unpack zero-points (same 8-nibble packing) and subtract
|
||||
# qzeros shape: [E, K//gs, N//8] int32
|
||||
gs = K // scale.shape[1]
|
||||
n_gs = scale.shape[1]
|
||||
zp_shifts = torch.arange(8, device=qzeros.device, dtype=torch.int32) * 4
|
||||
zp_nibbles = (qzeros.unsqueeze(-1) >> zp_shifts) & 0xF # [E, n_gs, N//8, 8]
|
||||
zp = zp_nibbles.reshape(E, n_gs, N).to(torch.int16) # [E, n_gs, N]
|
||||
zp = zp.repeat_interleave(gs, dim=1) # [E, K, N]
|
||||
w = w - zp
|
||||
|
||||
# Broadcast scale [E, K//gs, N] -> [E, K, N]
|
||||
gs = K // scale.shape[1]
|
||||
scale_broadcast = scale.repeat_interleave(gs, dim=1).to(output_dtype)
|
||||
|
||||
w_dequant = w.to(output_dtype) * scale_broadcast # [E, K, N]
|
||||
|
||||
if transpose_output:
|
||||
return w_dequant.permute(0, 2, 1).contiguous() # [E, N, K]
|
||||
return w_dequant.contiguous() # [E, K, N]
|
||||
|
||||
|
||||
def _unpack_and_dequant_int4_awq(
|
||||
w_int32: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
qzeros: torch.Tensor | None,
|
||||
transpose_output: bool,
|
||||
output_dtype: torch.dtype = torch.bfloat16,
|
||||
) -> torch.Tensor:
|
||||
"""Unpack AWQ-packed int4 weights and dequantize to output_dtype.
|
||||
|
||||
AWQ packs along the N (column) dimension with an interleave permutation
|
||||
[0,2,4,6,1,3,5,7] applied before packing, so unpacking must undo that.
|
||||
|
||||
Args:
|
||||
w_int32: packed weights, shape [E, K, N_packed] where N_packed = N//8
|
||||
(8 nibbles per int32, packed along N with AWQ interleaving).
|
||||
scale: per-group scales, shape [E, K//group_size, N], float16.
|
||||
qzeros: asymmetric zero-points, shape [E, K//gs, N_packed], int32.
|
||||
None for symmetric (uint4b8 with implicit bias 8).
|
||||
transpose_output: if True return [E, N, K]; if False return [E, K, N].
|
||||
output_dtype: target floating-point dtype (bfloat16 or float16).
|
||||
|
||||
Returns:
|
||||
Dequantized weight tensor in the requested layout.
|
||||
"""
|
||||
E, K, N_packed = w_int32.shape
|
||||
N = N_packed * 8
|
||||
|
||||
# Unpack 8 nibbles per int32 along the N dimension (LSB-first)
|
||||
shifts = torch.arange(8, device=w_int32.device, dtype=torch.int32) * 4
|
||||
# [E, K, N_packed, 8] -> [E, K, N_packed*8] = [E, K, N_interleaved]
|
||||
nibbles = (w_int32.unsqueeze(-1) >> shifts) & 0xF
|
||||
w_interleaved = nibbles.reshape(E, K, N) # [E, K, N] but column-interleaved
|
||||
|
||||
# Undo AWQ interleave: packed order is [0,2,4,6,1,3,5,7] within each group
|
||||
# of 8. Inverse: position i in packed -> original column interleave[i].
|
||||
# To reverse: we need the inverse permutation so that
|
||||
# w[:, :, inv_interleave] = w_interleaved gives the natural column order.
|
||||
interleave = torch.tensor([0, 2, 4, 6, 1, 3, 5, 7], device=w_int32.device)
|
||||
inv_interleave = torch.empty_like(interleave)
|
||||
inv_interleave[interleave] = torch.arange(8, device=w_int32.device)
|
||||
|
||||
# Apply inverse interleave within each group of 8 columns
|
||||
w_reshaped = w_interleaved.reshape(E, K, N // 8, 8) # [E, K, groups, 8]
|
||||
w_reordered = w_reshaped[:, :, :, inv_interleave] # undo interleave
|
||||
w = w_reordered.reshape(E, K, N).to(torch.int16) # [E, K, N]
|
||||
|
||||
if qzeros is None:
|
||||
w = w - 8
|
||||
else:
|
||||
# qzeros: [E, K//gs, N_packed] int32, same AWQ column packing
|
||||
gs = K // scale.shape[1]
|
||||
n_gs = scale.shape[1]
|
||||
zp_nibbles = (qzeros.unsqueeze(-1) >> shifts) & 0xF # [E, n_gs, N_packed, 8]
|
||||
zp_interleaved = zp_nibbles.reshape(E, n_gs, N)
|
||||
zp_reshaped = zp_interleaved.reshape(E, n_gs, N // 8, 8)
|
||||
zp_reordered = zp_reshaped[:, :, :, inv_interleave]
|
||||
zp = zp_reordered.reshape(E, n_gs, N).to(torch.int16) # [E, n_gs, N]
|
||||
zp = zp.repeat_interleave(gs, dim=1) # [E, K, N]
|
||||
w = w - zp
|
||||
|
||||
gs = K // scale.shape[1]
|
||||
scale_broadcast = scale.repeat_interleave(gs, dim=1).to(output_dtype) # [E, K, N]
|
||||
|
||||
w_dequant = w.to(output_dtype) * scale_broadcast # [E, K, N]
|
||||
|
||||
if transpose_output:
|
||||
return w_dequant.permute(0, 2, 1).contiguous() # [E, N, K]
|
||||
return w_dequant.contiguous() # [E, K, N]
|
||||
|
||||
|
||||
def _process_weights_emulation_gptq(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_qzeros: torch.Tensor | None,
|
||||
w2_qzeros: torch.Tensor | None,
|
||||
) -> tuple:
|
||||
"""Dequantize int4 weights to BF16 for the emulation backend.
|
||||
|
||||
Inputs are in GPTQ packed format:
|
||||
w13: [E, K//8, 2*N] int32 (gate+up proj stacked on dim 2)
|
||||
w2: [E, N//8, K] int32
|
||||
w13_scale: [E, K//gs, 2*N] float16
|
||||
w2_scale: [E, N//gs, K] float16
|
||||
|
||||
Outputs (what TritonExperts expects):
|
||||
w13_out: [E, 2*N, K] bfloat16
|
||||
w2_out: [E, K, N] bfloat16
|
||||
"""
|
||||
# w13: packed along K (dim 1), output cols are 2*N (dim 2)
|
||||
# transpose_output=True yields [E, 2*N, K]
|
||||
w13_bf16 = _unpack_and_dequant_int4_gptq(
|
||||
w13, w13_scale, w13_qzeros, transpose_output=True
|
||||
)
|
||||
|
||||
# w2: packed along N (dim 1 is N//8), output cols are K (dim 2)
|
||||
# After unpacking we get [E, N, K]; we want [E, K, N] for TritonExperts
|
||||
# transpose_output=False gives [E, N, K], then we permute once more
|
||||
w2_unpacked = _unpack_and_dequant_int4_gptq(
|
||||
w2, w2_scale, w2_qzeros, transpose_output=False
|
||||
) # [E, N, K]
|
||||
w2_bf16 = w2_unpacked.permute(0, 2, 1).contiguous() # [E, K, N]
|
||||
|
||||
dummy = torch.ones(1, dtype=torch.float16, device=w13.device)
|
||||
return (
|
||||
w13_bf16, # w13_qweight (now bf16, not int32)
|
||||
w2_bf16, # w2_qweight (now bf16, not int32)
|
||||
dummy, # w13_scales (unused; nulled out in Int4EmulationTritonExperts)
|
||||
dummy, # w2_scales (unused)
|
||||
None, # w13_g_idx
|
||||
None, # w2_g_idx
|
||||
None, # w13_g_idx_sort_indices
|
||||
None, # w2_g_idx_sort_indices
|
||||
None, # w13_qzeros
|
||||
None, # w2_qzeros
|
||||
None, # w13_input_global_scale
|
||||
None, # w2_input_global_scale
|
||||
None, # w13_bias
|
||||
None, # w2_bias
|
||||
)
|
||||
|
||||
|
||||
def _process_weights_emulation_awq(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_qzeros: torch.Tensor | None,
|
||||
w2_qzeros: torch.Tensor | None,
|
||||
) -> tuple:
|
||||
"""Dequantize AWQ int4 weights to BF16 for the emulation backend.
|
||||
|
||||
AWQ inputs:
|
||||
w13: [E, K, 2*N//8] int32 (packed along N, gate+up on dim 2)
|
||||
w2: [E, N, K//8] int32 (packed along K)
|
||||
w13_scale: [E, K//gs, 2*N] float16
|
||||
w2_scale: [E, N//gs, K] float16
|
||||
|
||||
Outputs (what TritonExperts expects):
|
||||
w13_out: [E, 2*N, K] bfloat16
|
||||
w2_out: [E, K, N] bfloat16
|
||||
"""
|
||||
# w13: AWQ-packed along N (dim 2), K is unpacked in dim 1
|
||||
# _unpack_and_dequant_int4_awq with transpose_output=True yields [E, 2*N, K]
|
||||
w13_bf16 = _unpack_and_dequant_int4_awq(
|
||||
w13, w13_scale, w13_qzeros, transpose_output=True
|
||||
)
|
||||
|
||||
# w2: AWQ packs along K (dim 2 is K//8), N is unpacked in dim 1.
|
||||
# AWQ w2 is [E, N, K//8] — same column-pack format applied to the K dim.
|
||||
# _unpack_and_dequant_int4_awq expects [E, rows, N_packed] where the
|
||||
# packed dim is columns. Treat dim 1 as rows and dim 2 as N_packed:
|
||||
# unpacking gives [E, N, K]. Then permute to [E, K, N].
|
||||
w2_unpacked = _unpack_and_dequant_int4_awq(
|
||||
w2, w2_scale, w2_qzeros, transpose_output=False
|
||||
) # [E, N, K]
|
||||
w2_bf16 = w2_unpacked.permute(0, 2, 1).contiguous() # [E, K, N]
|
||||
|
||||
dummy = torch.ones(1, dtype=torch.float16, device=w13.device)
|
||||
return (
|
||||
w13_bf16,
|
||||
w2_bf16,
|
||||
dummy,
|
||||
dummy,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def convert_to_wna16_moe_kernel_format(
|
||||
backend: WNA16MoEBackend,
|
||||
layer: torch.nn.Module,
|
||||
@@ -1203,5 +1462,25 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w13_bias_out,
|
||||
w2_bias_out,
|
||||
)
|
||||
elif backend == WNA16MoEBackend.EMULATION:
|
||||
from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig
|
||||
|
||||
if isinstance(quant_config, AutoAWQConfig):
|
||||
return _process_weights_emulation_awq(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
)
|
||||
return _process_weights_emulation_gptq(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported wna16 MoE backend: {backend.value}")
|
||||
|
||||
@@ -774,6 +774,9 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
a1_gscale=getattr(layer, "w13_input_global_scale", None),
|
||||
a2_gscale=getattr(layer, "w2_input_global_scale", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
|
||||
+17
@@ -142,6 +142,23 @@ class CompressedTensorsMoEMethod(FusedMoEMethodBase):
|
||||
return CompressedTensorsW4A16FlydslMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config
|
||||
)
|
||||
elif moe_backend == "emulation":
|
||||
# Although this is called 'Marlin', actually it selects
|
||||
# emulation backend by calling select_wna16_moe_backend.
|
||||
# TODO: we need to update CompressedTensorsWNA16MoeMethod
|
||||
# to honor "--moe-backend" option
|
||||
from .compressed_tensors_moe_wna16_marlin import (
|
||||
CompressedTensorsWNA16MarlinMoEMethod,
|
||||
)
|
||||
|
||||
logger.info_once(
|
||||
"Using CompressedTensorsWNA16MarlinMoEMethod "
|
||||
"(emulation backend requested)"
|
||||
)
|
||||
return CompressedTensorsWNA16MarlinMoEMethod(
|
||||
weight_quant, input_quant, layer.moe_config, layer_name
|
||||
)
|
||||
|
||||
from .compressed_tensors_moe_wna16 import (
|
||||
CompressedTensorsWNA16MoEMethod,
|
||||
)
|
||||
|
||||
+2
-1
@@ -108,7 +108,8 @@ class CompressedTensorsW4A16FlydslMoEMethod(CompressedTensorsMoEMethod):
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE"
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
|
||||
+2
-1
@@ -48,7 +48,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
# grouped actorder isn't supported by this kernel
|
||||
assert weight_quant.actorder != "group"
|
||||
assert weight_quant.symmetric, (
|
||||
"Only symmetric quantization is supported for MoE"
|
||||
"Only symmetric quantization is supported for MoE. "
|
||||
"Try --moe-backend emulation."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
|
||||
+18
-6
@@ -467,10 +467,16 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
if not is_flashinfer:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
if w13_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w13_weight_g_idx", w13_g_idx_processed)
|
||||
if w2_g_idx_processed is not None:
|
||||
replace_parameter(layer, "w2_weight_g_idx", w2_g_idx_processed)
|
||||
if w13_g_idx_sort_indices is not None:
|
||||
replace_parameter(
|
||||
layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices
|
||||
)
|
||||
if w2_g_idx_sort_indices is not None:
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
|
||||
# Register input global scales if present
|
||||
if w13_input_global_scale is not None:
|
||||
@@ -484,8 +490,11 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
# Marlin workspace — only needed for Marlin-family backends, not emulation.
|
||||
if (
|
||||
self.experts_cls is not None
|
||||
and issubclass(self.experts_cls, FusedMoEExpertsModular)
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device, 4
|
||||
@@ -528,6 +537,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
num_bits=self.num_bits,
|
||||
w1_zp=getattr(layer, "w13_weight_zero_point", None),
|
||||
w2_zp=getattr(layer, "w2_weight_zero_point", None),
|
||||
gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
Reference in New Issue
Block a user