[CPU][Zen] Route Int8 MoE inference through zentorch on AMD (#44834)
Signed-off-by: R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c35551f892
commit
8f03625b3d
@@ -18,6 +18,8 @@ steps:
|
||||
- vllm/model_executor/layers/attention/mla_attention.py
|
||||
- tests/kernels/moe/test_cpu_fused_moe.py
|
||||
- tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
- tests/kernels/moe/test_zen_cpu_int8_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
|
||||
- tests/kernels/test_onednn.py
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
@@ -36,6 +38,7 @@ steps:
|
||||
pytest -x -v -s tests/kernels/attention/test_amx_mla.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_zen_cpu_int8_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Zen CPU INT8 MoE experts (ZenCPUExpertsInt8)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.test_cpu_fused_moe import (
|
||||
_make_moe_config,
|
||||
quantize_per_channel,
|
||||
ref_fused_moe_int8,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
int8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ZenCPUExpertsInt8,
|
||||
select_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
Int8MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
EXPERT_NUM = 8
|
||||
HIDDEN_SIZE = 128
|
||||
INTERMEDIATE_SIZE = 128
|
||||
BATCH_SIZE = 32
|
||||
# zentorch names its activations as strings; the experts API is enum-typed.
|
||||
ZENTORCH_ACT = [act for act in MoEActivation if act.value in _ZENTORCH_MOE_ACTIVATIONS]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_zentorch_fused_moe():
|
||||
"""Reference zentorch_fused_moe, registered when zentorch is absent."""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_fused_moe"):
|
||||
yield None
|
||||
return
|
||||
|
||||
calls: dict[str, dict] = {}
|
||||
|
||||
def _fused_moe(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales=None,
|
||||
w2_scales=None,
|
||||
zentorch_op_name="",
|
||||
):
|
||||
calls["fused_moe"] = {
|
||||
"w13": w13,
|
||||
"w2": w2,
|
||||
"w13_bias": w13_bias,
|
||||
"w2_bias": w2_bias,
|
||||
"topk_weights": topk_weights,
|
||||
"topk_id": topk_id,
|
||||
"skip_weighted": skip_weighted,
|
||||
"act": act,
|
||||
"w13_scales": w13_scales,
|
||||
"w2_scales": w2_scales,
|
||||
}
|
||||
output.copy_(
|
||||
ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scales.float(),
|
||||
w2_scales.float(),
|
||||
None if w13_bias is None else w13_bias.float(),
|
||||
None if w2_bias is None else w2_bias.float(),
|
||||
topk_weights,
|
||||
topk_id,
|
||||
MoEActivation(act),
|
||||
)
|
||||
)
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define(
|
||||
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
|
||||
"Tensor topk_id, bool skip_weighted, str act, "
|
||||
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
|
||||
)
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl("zentorch_fused_moe", _fused_moe)
|
||||
|
||||
yield calls
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
def _make_int8_moe_weights(use_bias: bool, dtype: torch.dtype = torch.bfloat16):
|
||||
up_dim = 2 * INTERMEDIATE_SIZE
|
||||
w13_fp = torch.randn((EXPERT_NUM, up_dim, HIDDEN_SIZE), dtype=dtype) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
w2_fp = torch.randn((EXPERT_NUM, HIDDEN_SIZE, INTERMEDIATE_SIZE), dtype=dtype) / (
|
||||
0.5 * INTERMEDIATE_SIZE**0.5
|
||||
)
|
||||
w13, w13_scale = quantize_per_channel(w13_fp)
|
||||
w2, w2_scale = quantize_per_channel(w2_fp)
|
||||
w13_scale = w13_scale.to(torch.bfloat16).float()
|
||||
w2_scale = w2_scale.to(torch.bfloat16).float()
|
||||
|
||||
w13_bias = w2_bias = None
|
||||
if use_bias:
|
||||
w13_bias = torch.randn((EXPERT_NUM, up_dim), dtype=dtype) / (0.5 * up_dim**0.5)
|
||||
w2_bias = torch.randn((EXPERT_NUM, HIDDEN_SIZE), dtype=dtype) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
return w13, w2, w13_scale, w2_scale, w13_bias, w2_bias
|
||||
|
||||
|
||||
def test_zen_int8_first_in_cpu_backend_priority():
|
||||
"""The Zen experts are preferred over the generic CPU int8 experts."""
|
||||
assert backend_to_kernel_cls(Int8MoeBackend.CPU)[0] is ZenCPUExpertsInt8
|
||||
|
||||
|
||||
def test_zen_int8_support_predicates(mock_zentorch_fused_moe, monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: False)
|
||||
assert not ZenCPUExpertsInt8._supports_current_device()
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
assert ZenCPUExpertsInt8._supports_current_device()
|
||||
assert ZenCPUExpertsInt8._supports_quant_scheme(
|
||||
kInt8StaticChannelSym, kInt8DynamicTokenSym
|
||||
)
|
||||
assert not ZenCPUExpertsInt8._supports_quant_scheme(kInt8StaticChannelSym, None)
|
||||
for act in ZENTORCH_ACT:
|
||||
assert ZenCPUExpertsInt8._supports_activation(act)
|
||||
assert not ZenCPUExpertsInt8._supports_activation(MoEActivation.RELU2)
|
||||
assert not ZenCPUExpertsInt8._supports_activation(
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE
|
||||
)
|
||||
assert not ZenCPUExpertsInt8._supports_no_act_and_mul()
|
||||
assert ZenCPUExpertsInt8._supports_parallel_config(
|
||||
FusedMoEParallelConfig.make_no_parallel()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("act", ZENTORCH_ACT)
|
||||
@pytest.mark.parametrize("use_bias", [False, True])
|
||||
def test_zen_int8_dispatch_contract(
|
||||
default_vllm_config,
|
||||
mock_zentorch_fused_moe,
|
||||
act: MoEActivation,
|
||||
use_bias: bool,
|
||||
):
|
||||
"""apply() hands zentorch the weights, scales and biases it expects."""
|
||||
if mock_zentorch_fused_moe is None:
|
||||
pytest.skip("real zentorch is installed; the recorded call is unavailable")
|
||||
set_random_seed(0)
|
||||
|
||||
topk_num = EXPERT_NUM // 2
|
||||
w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = _make_int8_moe_weights(use_bias)
|
||||
moe_config = _make_moe_config(
|
||||
EXPERT_NUM,
|
||||
HIDDEN_SIZE,
|
||||
INTERMEDIATE_SIZE,
|
||||
topk_num,
|
||||
torch.bfloat16,
|
||||
act,
|
||||
use_bias,
|
||||
)
|
||||
# A layer carries per-channel scales as [E, N, 1].
|
||||
quant_config = int8_w8a8_moe_quant_config(
|
||||
w1_scale=w13_scale.unsqueeze(-1),
|
||||
w2_scale=w2_scale.unsqueeze(-1),
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
w1_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
per_act_token_quant=True,
|
||||
)
|
||||
|
||||
input = torch.randn((BATCH_SIZE, HIDDEN_SIZE), dtype=torch.bfloat16) / (
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16)
|
||||
|
||||
experts = ZenCPUExpertsInt8(moe_config, quant_config)
|
||||
# zentorch_fused_moe takes no expert_map.
|
||||
assert not experts.supports_expert_map()
|
||||
output = experts.apply(
|
||||
hidden_states=input,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
router_logits=router_logits,
|
||||
activation=act,
|
||||
global_num_experts=EXPERT_NUM,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
call = mock_zentorch_fused_moe["fused_moe"]
|
||||
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
assert call["w13_scales"].dtype == torch.bfloat16
|
||||
assert call["w2_scales"].dtype == torch.bfloat16
|
||||
assert call["act"] == act.value.lower()
|
||||
assert call["skip_weighted"] is False
|
||||
assert call["topk_weights"].dtype == torch.float32
|
||||
assert call["topk_id"].dtype == torch.int32
|
||||
if use_bias:
|
||||
assert call["w13_bias"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_bias"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
else:
|
||||
assert call["w13_bias"] is None and call["w2_bias"] is None
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=input,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=False,
|
||||
top_k=topk_num,
|
||||
renormalize=False,
|
||||
scoring_func="softmax",
|
||||
)
|
||||
ref = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
None if w13_bias is None else w13_bias.float(),
|
||||
None if w2_bias is None else w2_bias.float(),
|
||||
topk_weights.to(torch.float32),
|
||||
topk_ids.to(torch.int32),
|
||||
act,
|
||||
)
|
||||
torch.testing.assert_close(output, ref)
|
||||
@@ -20,7 +20,10 @@ from vllm._custom_ops import (
|
||||
cpu_prepack_moe_weight_int8,
|
||||
fused_experts_cpu,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
has_zentorch_op,
|
||||
is_zentorch_moe_config_supported,
|
||||
is_zentorch_moe_supported,
|
||||
)
|
||||
@@ -44,6 +47,7 @@ from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# ===========================================================================
|
||||
# Routing
|
||||
# ===========================================================================
|
||||
@@ -1382,3 +1386,161 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"neon",
|
||||
skip_weighted=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
|
||||
quantization, dispatched through zentorch."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
assert self.w1_scale is not None and self.w2_scale is not None, (
|
||||
"ZenCPUExpertsInt8 requires per-channel weight scales on the layer."
|
||||
)
|
||||
num_experts = self.w1_scale.shape[0]
|
||||
self._w1_scale_bf16 = (
|
||||
self.w1_scale.detach()
|
||||
.to(torch.bfloat16)
|
||||
.reshape(num_experts, -1)
|
||||
.contiguous()
|
||||
)
|
||||
self._w2_scale_bf16 = (
|
||||
self.w2_scale.detach()
|
||||
.to(torch.bfloat16)
|
||||
.reshape(num_experts, -1)
|
||||
.contiguous()
|
||||
)
|
||||
self._w1_bias_bf16 = (
|
||||
None
|
||||
if self.w1_bias is None
|
||||
else self.w1_bias.detach().to(torch.bfloat16).contiguous()
|
||||
)
|
||||
self._w2_bias_bf16 = (
|
||||
None
|
||||
if self.w2_bias is None
|
||||
else self.w2_bias.detach().to(torch.bfloat16).contiguous()
|
||||
)
|
||||
logger.info_once(
|
||||
"[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE (monolithic experts)"
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# zentorch_fused_moe quantizes activations itself.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return has_zentorch_op(["zentorch_fused_moe"])
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return str(activation.value).lower() in _ZENTORCH_MOE_ACTIVATIONS
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return not moe_parallel_config.use_ep
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return (weight_key, activation_key) == (
|
||||
kInt8StaticChannelSym,
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(
|
||||
routing_method: RoutingMethodType,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return routing_method in [
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _supports_router_logits_dtype(
|
||||
router_logits_dtype: torch.dtype | None,
|
||||
routing_method: RoutingMethodType,
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
hidden_states = hidden_states.mul(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
output = torch.empty_like(hidden_states)
|
||||
torch.ops.zentorch.zentorch_fused_moe(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
self._w1_bias_bf16,
|
||||
self._w2_bias_bf16,
|
||||
topk_weights.to(torch.float32).contiguous(),
|
||||
topk_ids.to(torch.int32).contiguous(),
|
||||
apply_router_weight_on_input, # skip_weighted
|
||||
str(activation.value).lower(),
|
||||
self._w1_scale_bf16,
|
||||
self._w2_scale_bf16,
|
||||
)
|
||||
return output
|
||||
|
||||
@@ -73,9 +73,10 @@ def backend_to_kernel_cls(
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUExpertsInt8,
|
||||
CPUExpertsInt8,
|
||||
ZenCPUExpertsInt8,
|
||||
)
|
||||
|
||||
return [ArmCPUExpertsInt8, CPUExpertsInt8]
|
||||
return [ZenCPUExpertsInt8, ArmCPUExpertsInt8, CPUExpertsInt8]
|
||||
else:
|
||||
raise ValueError(f"Unknown Int8 MoE backend: {backend.value}")
|
||||
|
||||
|
||||
+71
@@ -83,6 +83,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
# Capture bias dtype before params_dtype is overridden to int8 below.
|
||||
bias_dtype = params_dtype
|
||||
params_dtype = torch.int8
|
||||
w13_num_shards = 2 if self.moe.is_act_and_mul else 1
|
||||
|
||||
@@ -111,6 +113,26 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.register_parameter("w2_weight", w2_weight)
|
||||
set_weight_attrs(w2_weight, extra_weight_attrs)
|
||||
|
||||
# Per-expert biases (gpt-oss), allocated only when has_bias.
|
||||
if self.moe.has_bias:
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
w13_num_shards * intermediate_size_per_partition,
|
||||
dtype=bias_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w13_bias", w13_bias)
|
||||
set_weight_attrs(w13_bias, extra_weight_attrs)
|
||||
|
||||
w2_bias = torch.nn.Parameter(
|
||||
torch.zeros(num_experts, hidden_size, dtype=bias_dtype),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_bias", w2_bias)
|
||||
set_weight_attrs(w2_bias, extra_weight_attrs)
|
||||
|
||||
# WEIGHT_SCALES
|
||||
assert self.weight_quant.strategy == QuantizationStrategy.CHANNEL
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
@@ -141,6 +163,53 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.w2_input_scale = None
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# ZenDNN's swiglu_oai_mul reads gate/up interleaved (stride 2), not the
|
||||
# half-split layout _load_w13 leaves; reorder before the conversion below.
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ZenCPUExpertsInt8,
|
||||
)
|
||||
|
||||
_act = getattr(layer, "activation", None)
|
||||
_act_str = getattr(_act, "value", _act)
|
||||
if (
|
||||
self.experts_cls is ZenCPUExpertsInt8
|
||||
and isinstance(_act_str, str)
|
||||
and _act_str.lower() == "swigluoai"
|
||||
):
|
||||
two_i = layer.w13_weight.size(1)
|
||||
i = two_i // 2
|
||||
device = layer.w13_weight.device
|
||||
perm = torch.stack(
|
||||
[
|
||||
torch.arange(0, i, device=device),
|
||||
torch.arange(i, two_i, device=device),
|
||||
],
|
||||
dim=1,
|
||||
).flatten()
|
||||
has_w13_bias = getattr(layer, "w13_bias", None) is not None
|
||||
logger.info_once(
|
||||
"[zen_cpu][swigluoai-permute] Reordering w13 from "
|
||||
"half-split to interleaved for ZenDNN swiglu_oai_mul "
|
||||
"kernel: E=%d, 2I=%d, I=%d, has_w13_bias=%s",
|
||||
layer.w13_weight.size(0),
|
||||
two_i,
|
||||
i,
|
||||
has_w13_bias,
|
||||
)
|
||||
layer.w13_weight = torch.nn.Parameter(
|
||||
layer.w13_weight.data[:, perm, :].contiguous(),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.w13_weight_scale = torch.nn.Parameter(
|
||||
layer.w13_weight_scale.data[:, perm].contiguous(),
|
||||
requires_grad=False,
|
||||
)
|
||||
if has_w13_bias:
|
||||
layer.w13_bias = torch.nn.Parameter(
|
||||
layer.w13_bias.data[:, perm].contiguous(),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
w13, w2 = convert_to_int8_moe_kernel_format(
|
||||
int8_backend=self.int8_backend,
|
||||
w13=layer.w13_weight,
|
||||
@@ -169,6 +238,8 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
a1_scale=layer.w13_input_scale,
|
||||
a2_scale=layer.w2_input_scale,
|
||||
w1_bias=getattr(layer, "w13_bias", None),
|
||||
w2_bias=getattr(layer, "w2_bias", None),
|
||||
per_act_token_quant=True,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
@@ -288,7 +288,14 @@ class MixtralModel(nn.Module):
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
},
|
||||
orig_to_new_substr={
|
||||
# W8A8 compressed-tensors checkpoints name experts gate/up/down;
|
||||
# map to the w1/w3/w2 the FusedMoE loader expects (no-op if native).
|
||||
".gate_proj.": ".w1.",
|
||||
".up_proj.": ".w3.",
|
||||
".down_proj.": ".w2.",
|
||||
},
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
|
||||
Reference in New Issue
Block a user