diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index bf6b00377f..03561ceb8a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -554,23 +554,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): # =========================================================================== -def prepare_int8_moe_layer_for_cpu( - w13: torch.Tensor, - w2: torch.Tensor, -) -> tuple[torch.Tensor, torch.Tensor]: - """Prepack INT8 MoE weights for the current CPU architecture.""" - # SMMLA packing for AArch64 - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - return ( - cpu_prepack_moe_weight_int8(w13, "neon"), - cpu_prepack_moe_weight_int8(w2, "neon"), - ) - # VNNI packing for x86 - packed_w13 = torch.ops._C.convert_weight_packed(w13) - packed_w2 = torch.ops._C.convert_weight_packed(w2) - return packed_w13, packed_w2 - - class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic): """CPU INT8 W8A8 per-channel weight / dynamic per-token activation monolithic MoE experts.""" @@ -650,7 +633,8 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic): """VNNI-prepack INT8 MoE weights for CPU kernel.""" from vllm.model_executor.utils import replace_parameter - w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight) + w13 = torch.ops._C.convert_weight_packed(layer.w13_weight) + w2 = torch.ops._C.convert_weight_packed(layer.w2_weight) replace_parameter(layer, "w13_weight", w13) replace_parameter(layer, "w2_weight", w2) @@ -814,7 +798,8 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic): def process_weights_after_loading(self, layer: torch.nn.Module) -> None: from vllm.model_executor.utils import replace_parameter - w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight) + w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "neon") + w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "neon") replace_parameter(layer, "w13_weight", w13) replace_parameter(layer, "w2_weight", w2) diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index db484b9b31..ab7775bf32 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -266,13 +266,7 @@ def convert_to_int8_moe_kernel_format( quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale), ) return layer.w13_weight, layer.w2_weight - elif int8_backend == Int8MoeBackend.CPU: - from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( - prepare_int8_moe_layer_for_cpu, - ) - - w13, w2 = prepare_int8_moe_layer_for_cpu(w13, w2) - elif int8_backend != Int8MoeBackend.TRITON: + elif int8_backend not in (Int8MoeBackend.TRITON, Int8MoeBackend.CPU): raise ValueError(f"Unsupported Int8 MoE backend: {int8_backend.value}") return w13, w2 diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index d304ca56bf..28100b012e 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -162,6 +162,7 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): routing_tables=layer._expert_routing_tables(), layer=layer, ) + self.moe_kernel.fused_experts.process_weights_after_loading(layer) def maybe_make_prepare_finalize( self, diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py index ef76594164..870e4d2a0f 100644 --- a/vllm/model_executor/layers/quantization/online/int8.py +++ b/vllm/model_executor/layers/quantization/online/int8.py @@ -114,6 +114,7 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase): routing_tables=layer._expert_routing_tables(), layer=layer, ) + self.moe_kernel.fused_experts.process_weights_after_loading(layer) def get_fused_moe_quant_config( self, layer: torch.nn.Module