[chore] clean-up weight prepack for INT8 MoE (#50116)
Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
@@ -554,23 +554,6 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def prepare_int8_moe_layer_for_cpu(
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Prepack INT8 MoE weights for the current CPU architecture."""
|
||||
# SMMLA packing for AArch64
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
return (
|
||||
cpu_prepack_moe_weight_int8(w13, "neon"),
|
||||
cpu_prepack_moe_weight_int8(w2, "neon"),
|
||||
)
|
||||
# VNNI packing for x86
|
||||
packed_w13 = torch.ops._C.convert_weight_packed(w13)
|
||||
packed_w2 = torch.ops._C.convert_weight_packed(w2)
|
||||
return packed_w13, packed_w2
|
||||
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
monolithic MoE experts."""
|
||||
@@ -650,7 +633,8 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"""VNNI-prepack INT8 MoE weights for CPU kernel."""
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight)
|
||||
w13 = torch.ops._C.convert_weight_packed(layer.w13_weight)
|
||||
w2 = torch.ops._C.convert_weight_packed(layer.w2_weight)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
@@ -814,7 +798,8 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
w13, w2 = prepare_int8_moe_layer_for_cpu(layer.w13_weight, layer.w2_weight)
|
||||
w13 = cpu_prepack_moe_weight_int8(layer.w13_weight, "neon")
|
||||
w2 = cpu_prepack_moe_weight_int8(layer.w2_weight, "neon")
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
|
||||
@@ -266,13 +266,7 @@ def convert_to_int8_moe_kernel_format(
|
||||
quant_config=_humming_int8_weight_schema(w13, layer.w13_weight_scale),
|
||||
)
|
||||
return layer.w13_weight, layer.w2_weight
|
||||
elif int8_backend == Int8MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_int8_moe_layer_for_cpu,
|
||||
)
|
||||
|
||||
w13, w2 = prepare_int8_moe_layer_for_cpu(w13, w2)
|
||||
elif int8_backend != Int8MoeBackend.TRITON:
|
||||
elif int8_backend not in (Int8MoeBackend.TRITON, Int8MoeBackend.CPU):
|
||||
raise ValueError(f"Unsupported Int8 MoE backend: {int8_backend.value}")
|
||||
|
||||
return w13, w2
|
||||
|
||||
+1
@@ -162,6 +162,7 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
|
||||
@@ -114,6 +114,7 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
layer=layer,
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def get_fused_moe_quant_config(
|
||||
self, layer: torch.nn.Module
|
||||
|
||||
Reference in New Issue
Block a user