[CPU] Support CPU W4A16 INT4 MoE (#43409)
Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
This commit is contained in:
@@ -5,7 +5,12 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._custom_ops import CPUQuantMethod, fused_experts_cpu
|
||||
from vllm._custom_ops import (
|
||||
CPUQuantAlgo,
|
||||
CPUQuantMethod,
|
||||
convert_weight_packed_scale_zp,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -17,6 +22,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8Static128BlockSym,
|
||||
kInt4Static,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -318,3 +324,209 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
limit,
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
|
||||
def prepare_int4_moe_layer_for_cpu(
|
||||
w13_packed: torch.Tensor,
|
||||
w2_packed: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
quant_algo: CPUQuantAlgo = CPUQuantAlgo.GPTQ,
|
||||
w13_zeros: torch.Tensor | None = None,
|
||||
w2_zeros: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Repack INT4 MoE weights via convert_weight_packed_scale_zp for CPU.
|
||||
|
||||
Args:
|
||||
w13_packed: [E, K//8, 2*I] int32 (packed int4)
|
||||
w2_packed: [E, I//8, K] int32 (packed int4)
|
||||
w13_scale: [E, num_groups, 2*I] float16/bf16
|
||||
w2_scale: [E, num_groups, K] float16/bf16
|
||||
quant_algo: CPUQuantAlgo.GPTQ or CPUQuantAlgo.AWQ
|
||||
w13_zeros: optional [E, num_groups, N//8] int32 packed zeros.
|
||||
If None, synthetic zeros are created for symmetric quant.
|
||||
w2_zeros: optional [E, num_groups, N//8] int32 packed zeros.
|
||||
If None, synthetic zeros are created for symmetric quant.
|
||||
|
||||
Returns:
|
||||
(blocked_w13, blocked_w2, blocked_s13, blocked_s2,
|
||||
blocked_z13, blocked_z2)
|
||||
"""
|
||||
E = w13_packed.size(0)
|
||||
|
||||
# No qzeros are available in compressed-tensors symmetric checkpoints.
|
||||
# The GPTQ unpack kernel (unpack_4bit_to_32bit_signed) adds +1 to stored zeros,
|
||||
# so we store 7 per nibble: 0x77777777 → +1 → 8.
|
||||
if w13_zeros is None:
|
||||
num_groups_w13 = w13_scale.size(1)
|
||||
N_w13 = w13_scale.size(2) # 2*I
|
||||
_zp = 0x77777777
|
||||
w13_zeros = torch.full(
|
||||
(E, num_groups_w13, N_w13 // 8),
|
||||
_zp,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
if w2_zeros is None:
|
||||
num_groups_w2 = w2_scale.size(1)
|
||||
N_w2 = w2_scale.size(2) # K
|
||||
_zp = 0x77777777
|
||||
w2_zeros = torch.full(
|
||||
(E, num_groups_w2, N_w2 // 8),
|
||||
_zp,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
blocked_w13, blocked_z13, blocked_s13 = convert_weight_packed_scale_zp(
|
||||
w13_packed, w13_zeros, w13_scale, quant_algo
|
||||
)
|
||||
blocked_w2, blocked_z2, blocked_s2 = convert_weight_packed_scale_zp(
|
||||
w2_packed, w2_zeros, w2_scale, quant_algo
|
||||
)
|
||||
return (blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
|
||||
|
||||
|
||||
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU INT4 W4A16 group-quantized monolithic MoE experts.
|
||||
|
||||
Weights are int4 (packed), activations are bf16/fp16.
|
||||
Internally uses int8 compute via fused_experts_cpu with INT4_W4A8.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
moe_config,
|
||||
quant_config,
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cpu()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SILU
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kInt4Static, None),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsInt4 (W4A16) does not support "
|
||||
"apply_router_weight_on_input=True. "
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
return fused_experts_cpu(
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.INT4_W4A8,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
self.w1_zp,
|
||||
self.w2_zp,
|
||||
None, # block_size
|
||||
None, # w1_bias
|
||||
None, # w2_bias
|
||||
None, # alpha
|
||||
None, # limit
|
||||
True, # is_vnni
|
||||
)
|
||||
|
||||
@@ -45,6 +45,7 @@ logger = init_logger(__name__)
|
||||
class WNA16MoEBackend(Enum):
|
||||
MARLIN = "MARLIN"
|
||||
BATCHED_MARLIN = "BATCHED_MARLIN"
|
||||
CPU = "CPU"
|
||||
FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM"
|
||||
XPU = "XPU"
|
||||
|
||||
@@ -65,6 +66,12 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [XPUExpertsWNA16]
|
||||
elif backend == WNA16MoEBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
|
||||
return [CPUExpertsInt4]
|
||||
else:
|
||||
raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}")
|
||||
|
||||
@@ -73,6 +80,8 @@ def _get_priority_backends() -> list[WNA16MoEBackend]:
|
||||
"""
|
||||
Get available backends in priority order based on platform and config.
|
||||
"""
|
||||
if current_platform.is_cpu():
|
||||
return [WNA16MoEBackend.CPU]
|
||||
if current_platform.is_xpu():
|
||||
return [WNA16MoEBackend.XPU]
|
||||
|
||||
@@ -210,17 +219,21 @@ def make_wna16_moe_kernel(
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsWNA16,
|
||||
)
|
||||
|
||||
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts
|
||||
# and BatchedMarlinExperts
|
||||
# Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts,
|
||||
# BatchedMarlinExperts, XPUExpertsWNA16, and CPUExpertsInt4
|
||||
assert experts_cls in (
|
||||
MarlinExperts,
|
||||
BatchedMarlinExperts,
|
||||
TrtLlmMxint4ExpertsMonolithic,
|
||||
XPUExpertsWNA16,
|
||||
CPUExpertsInt4,
|
||||
)
|
||||
|
||||
is_monolithic = experts_cls.is_monolithic()
|
||||
@@ -683,6 +696,117 @@ def _process_awq_weights_marlin(
|
||||
)
|
||||
|
||||
|
||||
def _process_weights_cpu(
|
||||
quant_config: QuantizationConfig | QuantizationArgs | None,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
w13_g_idx: torch.Tensor | None = None,
|
||||
w2_g_idx: torch.Tensor | None = None,
|
||||
w13_qzeros: torch.Tensor | None = None,
|
||||
w2_qzeros: torch.Tensor | None = None,
|
||||
w13_bias: torch.Tensor | None = None,
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, # w13_qweight
|
||||
torch.Tensor, # w2_qweight
|
||||
torch.Tensor, # w13_scales
|
||||
torch.Tensor, # w2_scales
|
||||
torch.Tensor | None, # w13_g_idx
|
||||
torch.Tensor | None, # w2_g_idx
|
||||
torch.Tensor | None, # w13_g_idx_sort_indices
|
||||
torch.Tensor | None, # w2_g_idx_sort_indices
|
||||
torch.Tensor | None, # w13_qzeros
|
||||
torch.Tensor | None, # w2_qzeros
|
||||
torch.Tensor | None, # w13_input_global_scale
|
||||
torch.Tensor | None, # w2_input_global_scale
|
||||
torch.Tensor | None, # w13_bias
|
||||
torch.Tensor | None, # w2_bias
|
||||
]:
|
||||
"""CPU INT4 W4A16 weight post-processing."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_int4_moe_layer_for_cpu,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.awq_marlin import (
|
||||
AWQMarlinConfig,
|
||||
)
|
||||
|
||||
# Detect packing format.
|
||||
# AWQ: qweight is [E, K, 2*N//8] (packed along output/N dim).
|
||||
# GPTQ: qweight is [E, K//8, 2*N] (packed along input/K dim).
|
||||
# compressed-tensors: qweight is [E, K//8, 2*N] (packed along input/K dim).
|
||||
if isinstance(quant_config, AWQMarlinConfig):
|
||||
# AWQ: K is stored unpacked in dim 1.
|
||||
cpu_quant_algo = ops.CPUQuantAlgo.AWQ
|
||||
elif isinstance(quant_config, (AutoGPTQConfig, QuantizationArgs)):
|
||||
# GPTQ / compressed-tensors: K//8 is stored packed in dim 1.
|
||||
if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act:
|
||||
raise NotImplementedError(
|
||||
"CPU WNA16 MoE backend does not support GPTQ with "
|
||||
"desc_act=True. The fused MoE kernel has no g_idx "
|
||||
"reordering support."
|
||||
)
|
||||
cpu_quant_algo = ops.CPUQuantAlgo.GPTQ
|
||||
else:
|
||||
raise TypeError(
|
||||
"CPU WNA16 MoE backend requires AWQMarlinConfig, AutoGPTQConfig "
|
||||
f"or QuantizationArgs, got {type(quant_config).__name__}."
|
||||
)
|
||||
|
||||
# Determine zero points for repacking.
|
||||
w13_zeros: torch.Tensor | None = None
|
||||
w2_zeros: torch.Tensor | None = None
|
||||
if w13_qzeros is not None:
|
||||
w13_zeros = (
|
||||
w13_qzeros.data.view(torch.int32)
|
||||
if w13_qzeros.dtype != torch.int32
|
||||
else w13_qzeros.data
|
||||
)
|
||||
if w2_qzeros is not None:
|
||||
w2_zeros = (
|
||||
w2_qzeros.data.view(torch.int32)
|
||||
if w2_qzeros.dtype != torch.int32
|
||||
else w2_qzeros.data
|
||||
)
|
||||
|
||||
(
|
||||
blocked_w13,
|
||||
blocked_w2,
|
||||
blocked_s13,
|
||||
blocked_s2,
|
||||
blocked_z13,
|
||||
blocked_z2,
|
||||
) = prepare_int4_moe_layer_for_cpu(
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
quant_algo=cpu_quant_algo,
|
||||
w13_zeros=w13_zeros,
|
||||
w2_zeros=w2_zeros,
|
||||
)
|
||||
return (
|
||||
blocked_w13,
|
||||
blocked_w2,
|
||||
blocked_s13,
|
||||
blocked_s2,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
None, # w13_g_idx_sort_indices (unused on CPU)
|
||||
None, # w2_g_idx_sort_indices (unused on CPU)
|
||||
blocked_z13,
|
||||
blocked_z2,
|
||||
None, # w13_input_global_scale
|
||||
None, # w2_input_global_scale
|
||||
w13_bias.to(torch.float32) if w13_bias is not None else None,
|
||||
w2_bias.to(torch.float32) if w2_bias is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _process_weights_xpu(
|
||||
layer: torch.nn.Module,
|
||||
quant_config: QuantizationConfig,
|
||||
@@ -857,6 +981,20 @@ def convert_to_wna16_moe_kernel_format(
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif backend == WNA16MoEBackend.CPU:
|
||||
return _process_weights_cpu(
|
||||
quant_config,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_g_idx,
|
||||
w2_g_idx,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM:
|
||||
return _process_weights_flashinfer(
|
||||
w13,
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEExpertsModular,
|
||||
FusedMoEMethodBase,
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -640,8 +641,11 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
layer.register_parameter("w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
set_weight_attrs(w2_g_idx_sort_indices, extra_weight_attrs)
|
||||
|
||||
device = layer.w13_qweight.device
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
):
|
||||
device = layer.w13_qweight.device
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1
|
||||
@@ -660,8 +664,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2_g_idx,
|
||||
w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices,
|
||||
_w13_qzeros,
|
||||
_w2_qzeros,
|
||||
w13_qzeros,
|
||||
w2_qzeros,
|
||||
w13_input_global_scale,
|
||||
w2_input_global_scale,
|
||||
w13_bias,
|
||||
@@ -689,6 +693,10 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, "w2_g_idx", w2_g_idx)
|
||||
replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices)
|
||||
replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices)
|
||||
if w13_qzeros is not None:
|
||||
replace_parameter(layer, "w13_qzeros", w13_qzeros)
|
||||
if w2_qzeros is not None:
|
||||
replace_parameter(layer, "w2_qzeros", w2_qzeros)
|
||||
if w13_input_global_scale is not None:
|
||||
if hasattr(layer, "w13_input_global_scale"):
|
||||
replace_parameter(
|
||||
@@ -735,8 +743,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=layer.w13_g_idx,
|
||||
w2_g_idx=layer.w2_g_idx,
|
||||
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
@@ -750,12 +758,8 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=layer.w2_scales,
|
||||
weight_bits=self.quant_config.weight_bits,
|
||||
group_size=self.quant_config.group_size,
|
||||
w1_zp=getattr(layer, "w13_qzeros", None)
|
||||
if not self.quant_config.is_sym
|
||||
else None,
|
||||
w2_zp=getattr(layer, "w2_qzeros", None)
|
||||
if not self.quant_config.is_sym
|
||||
else None,
|
||||
w1_zp=getattr(layer, "w13_qzeros", None),
|
||||
w2_zp=getattr(layer, "w2_qzeros", None),
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
)
|
||||
@@ -794,3 +798,27 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
@@ -700,8 +700,8 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
is_k_full=self.is_k_full,
|
||||
w13_g_idx=getattr(layer, "w13_g_idx", None),
|
||||
w2_g_idx=getattr(layer, "w2_g_idx", None),
|
||||
w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices,
|
||||
w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices,
|
||||
w13_g_idx_sort_indices=getattr(layer, "w13_g_idx_sort_indices", None),
|
||||
w2_g_idx_sort_indices=getattr(layer, "w2_g_idx_sort_indices", None),
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
@@ -757,3 +757,27 @@ class AWQMarlinMoEMethod(FusedMoEMethodBase):
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
x: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply_monolithic(
|
||||
hidden_states=x,
|
||||
w1=layer.w13_qweight,
|
||||
w2=layer.w2_qweight,
|
||||
router_logits=router_logits,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
)
|
||||
|
||||
+9
-4
@@ -10,6 +10,7 @@ from compressed_tensors.quantization import (
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEExpertsModular,
|
||||
RoutedExperts,
|
||||
SharedExperts,
|
||||
)
|
||||
@@ -414,8 +415,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scales)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scales)
|
||||
|
||||
if not self.symmetric:
|
||||
if w13_qzeros is not None:
|
||||
replace_parameter(layer, "w13_weight_zero_point", w13_qzeros)
|
||||
if w2_qzeros is not None:
|
||||
replace_parameter(layer, "w2_weight_zero_point", w2_qzeros)
|
||||
|
||||
# Marlin-specific parameters (not needed for Flashinfer)
|
||||
@@ -437,9 +439,12 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
torch.nn.Parameter(w2_input_global_scale, requires_grad=False),
|
||||
)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device, 4
|
||||
)
|
||||
if self.experts_cls is not None and issubclass(
|
||||
self.experts_cls, FusedMoEExpertsModular
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device, 4
|
||||
)
|
||||
|
||||
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
|
||||
layer.w13_weight = layer.w13_weight_packed
|
||||
|
||||
Reference in New Issue
Block a user