[CPU] Migrate unquantized MoE to the modular-kernel experts structure (#50133)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Fadi Arafeh <fadi.arafeh@arm.com>
This commit is contained in:
co-authored by
Claude Opus 5
Fadi Arafeh
parent
b3f97dae24
commit
0a6446005d
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.allclose_default import get_default_atol, get_default_rtol
|
||||
from vllm._custom_ops import (
|
||||
cpu_fused_moe,
|
||||
@@ -12,9 +15,18 @@ from vllm._custom_ops import (
|
||||
cpu_prepack_moe_weight_int8,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import (
|
||||
_CPU_MOE_ACT_FN,
|
||||
CPUFusedMOE,
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
biased_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
select_experts,
|
||||
)
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -36,7 +48,10 @@ ACT = [
|
||||
]
|
||||
USE_BIAS = [False, True]
|
||||
ISA = ["vec"]
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
if (
|
||||
current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
and sys.platform != "darwin"
|
||||
):
|
||||
ISA.append("neon")
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
ISA.append("amx")
|
||||
@@ -44,6 +59,27 @@ if torch.cpu._is_amx_tile_supported():
|
||||
DTYPE = [torch.bfloat16]
|
||||
|
||||
|
||||
def _ref_moe_activation(
|
||||
input: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
) -> torch.Tensor:
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
gate = input[..., ::2].clamp(max=7.0)
|
||||
up = input[..., 1::2].clamp(min=-7.0, max=7.0)
|
||||
glu = gate * torch.sigmoid(gate * 1.702)
|
||||
return (up + 1) * glu
|
||||
|
||||
d = input.shape[-1] // 2
|
||||
gate, up = input[..., :d], input[..., d:]
|
||||
if activation == MoEActivation.SILU:
|
||||
return torch.nn.functional.silu(gate) * up
|
||||
if activation == MoEActivation.GELU:
|
||||
return torch.nn.functional.gelu(gate, approximate="none") * up
|
||||
if activation == MoEActivation.GELU_TANH:
|
||||
return torch.nn.functional.gelu(gate, approximate="tanh") * up
|
||||
raise ValueError(f"Unsupported activation: {activation}")
|
||||
|
||||
|
||||
def ref_fused_moe(
|
||||
input: torch.Tensor,
|
||||
w13: torch.Tensor,
|
||||
@@ -87,7 +123,8 @@ def ref_fused_moe(
|
||||
tokens_for_this_expert, curr_w13, curr_w13_bias
|
||||
)
|
||||
# Note: to simulate the kernel implementation
|
||||
gate_up = _CPU_MOE_ACT_FN[activation](gate_up).to(dtype=input.dtype).float()
|
||||
gate_up = _ref_moe_activation(gate_up, activation)
|
||||
gate_up = gate_up.to(dtype=input.dtype).float()
|
||||
expert_out = torch.nn.functional.linear(gate_up, curr_w2, curr_w2_bias)
|
||||
|
||||
outputs.append(expert_out)
|
||||
@@ -166,7 +203,7 @@ def ref_fused_moe_int8(
|
||||
if w13_bias is not None:
|
||||
gate_up += w13_bias[expert_idx].float()
|
||||
|
||||
intermediate = _CPU_MOE_ACT_FN[activation](gate_up).to(input.dtype)
|
||||
intermediate = _ref_moe_activation(gate_up, activation).to(input.dtype)
|
||||
intermediate_int8, intermediate_scale = quantize_per_token(intermediate)
|
||||
output = torch.matmul(
|
||||
intermediate_int8.float(),
|
||||
@@ -356,8 +393,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
|
||||
|
||||
class _StubMoELayer(torch.nn.Module):
|
||||
"""Minimal stand-in for the real MoE layer module, exposing just what
|
||||
CPUFusedMOE reads/replaces (w13_weight, w2_weight, activation, and
|
||||
optionally w13_bias/w2_bias)."""
|
||||
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
|
||||
router configuration, and optionally w13_bias/w2_bias)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -375,12 +412,77 @@ class _StubMoELayer(torch.nn.Module):
|
||||
self.w13_bias = torch.nn.Parameter(w13_bias, requires_grad=False)
|
||||
if w2_bias is not None:
|
||||
self.w2_bias = torch.nn.Parameter(w2_bias, requires_grad=False)
|
||||
self.use_grouped_topk = False
|
||||
self.renormalize = False
|
||||
self.scoring_func = "softmax"
|
||||
self.custom_routing_function = None
|
||||
|
||||
|
||||
def _make_moe_config(
|
||||
expert_num: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
topk_num: int,
|
||||
dtype: torch.dtype,
|
||||
act: MoEActivation,
|
||||
has_bias: bool = False,
|
||||
) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=expert_num,
|
||||
experts_per_token=topk_num,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
num_local_experts=expert_num,
|
||||
num_logical_experts=expert_num,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=act,
|
||||
in_dtype=dtype,
|
||||
device="cpu",
|
||||
routing_method=RoutingMethodType.Default,
|
||||
has_bias=has_bias,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("hidden_size", "intermediate_size", "act", "expected"),
|
||||
[
|
||||
(128, 128, MoEActivation.SILU, True),
|
||||
# Hidden size is not 32-aligned.
|
||||
(112, 128, MoEActivation.SILU, False),
|
||||
(128, 176, MoEActivation.SILU, True),
|
||||
# SwiGLUOAI's interleaved gate/up layout cannot be padded.
|
||||
(128, 176, MoEActivation.SWIGLUOAI, False),
|
||||
],
|
||||
)
|
||||
def test_cpu_vec_fused_moe_shape_support(
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
act: MoEActivation,
|
||||
expected: bool,
|
||||
):
|
||||
moe_config = _make_moe_config(
|
||||
expert_num=8,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
topk_num=4,
|
||||
dtype=torch.bfloat16,
|
||||
act=act,
|
||||
)
|
||||
supported, _ = CPUUnquantizedExperts.is_supported_config(
|
||||
CPUUnquantizedExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expert_num", EXPERT_NUM)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_DIM)
|
||||
@pytest.mark.parametrize("use_bias", USE_BIAS)
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
@pytest.mark.parametrize("isa", ISA)
|
||||
@pytest.mark.parametrize(
|
||||
"act",
|
||||
[MoEActivation.SILU, MoEActivation.GELU, MoEActivation.GELU_TANH],
|
||||
@@ -391,14 +493,11 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
hidden_size: int,
|
||||
use_bias: bool,
|
||||
dtype: torch.dtype,
|
||||
isa: str,
|
||||
act: MoEActivation,
|
||||
):
|
||||
"""An unaligned per-partition moe_intermediate_size must still hit the
|
||||
grouped-gemm fast path via automatic zero-padding, with numerically
|
||||
correct output, instead of silently falling back to the much slower
|
||||
per-expert torch loop."""
|
||||
if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
|
||||
pytest.skip("padding is only applied on the x86 AMX/vector kernels")
|
||||
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
|
||||
weights before prepacking."""
|
||||
|
||||
set_random_seed(0)
|
||||
batch_size = 64
|
||||
@@ -423,9 +522,15 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
score = torch.softmax(router_logits, dim=-1, dtype=torch.float32)
|
||||
topk_weight, topk_ids = torch.topk(score, topk_num)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
# Route with the same helper apply() uses internally, so the reference
|
||||
# only differs from the kernel in how the experts are evaluated.
|
||||
topk_weight, topk_ids = select_experts(
|
||||
hidden_states=input,
|
||||
router_logits=router_logits,
|
||||
top_k=topk_num,
|
||||
use_grouped_topk=False,
|
||||
renormalize=False,
|
||||
)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
|
||||
@@ -439,40 +544,48 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
w2_bias.clone() if w2_bias is not None else None,
|
||||
)
|
||||
|
||||
cpu_moe = CPUFusedMOE(layer)
|
||||
assert cpu_moe.forward_method == cpu_moe.forward_grouped_gemm, (
|
||||
"expected the padded intermediate size to hit the grouped-gemm fast path"
|
||||
moe_config = _make_moe_config(
|
||||
expert_num, hidden_size, intermediate_size, topk_num, dtype, act, use_bias
|
||||
)
|
||||
experts_cls = {
|
||||
"vec": CPUUnquantizedExperts,
|
||||
"amx": X86CPUUnquantizedExperts,
|
||||
"neon": ArmCPUUnquantizedExperts,
|
||||
}[isa]
|
||||
supported, reason = experts_cls.is_supported_config(
|
||||
experts_cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, (
|
||||
f"expected the padded intermediate size to be supported, got {reason}"
|
||||
)
|
||||
|
||||
output = cpu_moe.forward_method(
|
||||
layer, input, topk_weight, topk_ids, act, expert_num, False
|
||||
# Mirror UnquantizedFusedMoEMethod._setup_kernel: the quant config is
|
||||
# built from the layer first, then the experts shuffle the layer into the
|
||||
# kernel's runtime format.
|
||||
quant_config = (
|
||||
biased_moe_quant_config(layer.w13_bias, layer.w2_bias)
|
||||
if use_bias
|
||||
else FusedMoEQuantConfig.make()
|
||||
)
|
||||
experts = experts_cls(moe_config, quant_config)
|
||||
assert experts.isa == isa
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
output = experts.apply(
|
||||
hidden_states=input,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
activation=act,
|
||||
global_num_experts=expert_num,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
atol, rtol = get_default_atol(output), get_default_rtol(output)
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_cpu_fused_moe_unaligned_intermediate_size_swigluoai(default_vllm_config):
|
||||
"""swigluoai's interleaved gate/up layout isn't padded automatically. On
|
||||
AMX-capable CPUs this must raise rather than silently falling back to
|
||||
the (correct but much slower) per-expert torch loop; elsewhere it should
|
||||
fall back exactly as before."""
|
||||
set_random_seed(0)
|
||||
expert_num = 8
|
||||
hidden_size = 128
|
||||
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
|
||||
dtype = torch.bfloat16
|
||||
up_dim = 2 * intermediate_size
|
||||
|
||||
layer = _StubMoELayer(
|
||||
torch.randn((expert_num, up_dim, hidden_size), dtype=dtype),
|
||||
torch.randn((expert_num, hidden_size, intermediate_size), dtype=dtype),
|
||||
MoEActivation.SWIGLUOAI,
|
||||
)
|
||||
|
||||
if torch.cpu._is_amx_tile_supported():
|
||||
with pytest.raises(RuntimeError):
|
||||
CPUFusedMOE(layer)
|
||||
else:
|
||||
cpu_moe = CPUFusedMOE(layer)
|
||||
assert cpu_moe.forward_method == cpu_moe.forward_torch
|
||||
|
||||
@@ -1482,85 +1482,6 @@ def test_moe_sum_pad_aware(topk: int, dtype: torch.dtype, topk_ids_dtype: torch.
|
||||
opcheck(torch.ops._moe_C.moe_sum, (input, actual, topk_ids, expert_map))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("m", [1, 33])
|
||||
@pytest.mark.parametrize("n,k", [(128, 128)])
|
||||
@pytest.mark.parametrize("e", [8])
|
||||
@pytest.mark.parametrize("topk", [2])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("with_bias", [False, True])
|
||||
@pytest.mark.parametrize("activation", [MoEActivation.SILU])
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only test")
|
||||
def test_cpu_fused_moe_basic(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
e: int,
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
with_bias: bool,
|
||||
activation: MoEActivation,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.cpu_fused_moe import CPUFusedMOE
|
||||
|
||||
device = "cpu"
|
||||
set_random_seed(7)
|
||||
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
w13 = torch.randn((e, 2 * n, k), device=device, dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device=device, dtype=dtype) / 10
|
||||
router_logits = torch.randn((m, e), device=device, dtype=dtype)
|
||||
|
||||
b1 = b2 = None
|
||||
if with_bias:
|
||||
b1 = torch.randn((e, 2 * n), device=device, dtype=dtype) / 10
|
||||
b2 = torch.randn((e, k), device=device, dtype=dtype) / 10
|
||||
|
||||
ref = (
|
||||
torch_moe(a, w13, w2, router_logits, topk, b1, b2)
|
||||
if with_bias
|
||||
else torch_moe(a, w13, w2, router_logits, topk)
|
||||
)
|
||||
|
||||
class _Dummy(torch.nn.Module):
|
||||
def __init__(self, w13, w2, b1=None, b2=None):
|
||||
super().__init__()
|
||||
self.w13_weight = torch.nn.Parameter(w13, requires_grad=False)
|
||||
self.w2_weight = torch.nn.Parameter(w2, requires_grad=False)
|
||||
if b1 is not None:
|
||||
self.w13_bias = torch.nn.Parameter(b1, requires_grad=False)
|
||||
if b2 is not None:
|
||||
self.w2_bias = torch.nn.Parameter(b2, requires_grad=False)
|
||||
|
||||
layer = _Dummy(w13, w2, b1, b2).to(dtype)
|
||||
fused = CPUFusedMOE(layer)
|
||||
out = fused(
|
||||
layer=layer,
|
||||
x=a,
|
||||
use_grouped_topk=False,
|
||||
top_k=topk,
|
||||
router_logits=router_logits,
|
||||
renormalize=False,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
custom_routing_function=None,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=1.0,
|
||||
e_score_correction_bias=None,
|
||||
apply_router_weight_on_input=False,
|
||||
activation=activation,
|
||||
)
|
||||
|
||||
# Tolerances: fp32 tight; bf16 looser (esp. with bias)
|
||||
if dtype == torch.float32:
|
||||
atol = 1e-3
|
||||
elif with_bias:
|
||||
atol = 8e-2
|
||||
else:
|
||||
atol = 5e-2
|
||||
torch.testing.assert_close(out, ref, atol=atol, rtol=0)
|
||||
|
||||
|
||||
def _batched_fused_marlin_moe_cases() -> list[Any]:
|
||||
cases = [
|
||||
pytest.param(
|
||||
|
||||
@@ -3,14 +3,16 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
select_unquantized_moe_backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
skipif_not_cuda_rocm = pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm()),
|
||||
@@ -18,6 +20,109 @@ skipif_not_cuda_rocm = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("amx_supported", "in_dtype", "expect_amx_kernel"),
|
||||
[
|
||||
(True, torch.bfloat16, True),
|
||||
(False, torch.bfloat16, False),
|
||||
(True, torch.float16, False),
|
||||
],
|
||||
)
|
||||
def test_x86_cpu_unquantized_kernel_selection(
|
||||
amx_supported: bool,
|
||||
in_dtype: torch.dtype,
|
||||
expect_amx_kernel: bool,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "is_cpu", return_value=True),
|
||||
patch.object(
|
||||
current_platform,
|
||||
"get_cpu_architecture",
|
||||
return_value=CpuArchEnum.X86,
|
||||
),
|
||||
patch("torch.cpu._is_amx_tile_supported", return_value=amx_supported),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
in_dtype=in_dtype,
|
||||
)
|
||||
kernel_cls = next(
|
||||
cls
|
||||
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
|
||||
if cls.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
CPUUnquantizedExperts.activation_format(),
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected_kernel_cls = (
|
||||
X86CPUUnquantizedExperts if expect_amx_kernel else CPUUnquantizedExperts
|
||||
)
|
||||
assert kernel_cls is expected_kernel_cls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("platform", "in_dtype", "expect_arm_kernel"),
|
||||
[
|
||||
("linux", torch.bfloat16, True),
|
||||
("linux", torch.float16, False),
|
||||
("darwin", torch.bfloat16, False),
|
||||
],
|
||||
)
|
||||
def test_arm_cpu_unquantized_kernel_selection(
|
||||
platform: str,
|
||||
in_dtype: torch.dtype,
|
||||
expect_arm_kernel: bool,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ArmCPUUnquantizedExperts,
|
||||
CPUUnquantizedExperts,
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(current_platform, "is_cpu", return_value=True),
|
||||
patch.object(
|
||||
current_platform,
|
||||
"get_cpu_architecture",
|
||||
return_value=CpuArchEnum.ARM,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.cpu_moe.sys.platform",
|
||||
platform,
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
in_dtype=in_dtype,
|
||||
)
|
||||
kernel_cls = next(
|
||||
cls
|
||||
for cls in backend_to_kernel_cls(UnquantizedMoeBackend.CPU)
|
||||
if cls.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
CPUUnquantizedExperts.activation_format(),
|
||||
)[0]
|
||||
)
|
||||
|
||||
expected_kernel_cls = (
|
||||
ArmCPUUnquantizedExperts if expect_arm_kernel else CPUUnquantizedExperts
|
||||
)
|
||||
assert kernel_cls is expected_kernel_cls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"platform_method,expected_backend",
|
||||
[
|
||||
@@ -69,14 +174,20 @@ def test_select_default_backend_by_platform(
|
||||
patch.object(current_platform, "is_out_of_tree", return_value=False),
|
||||
patch.object(current_platform, platform_method, return_value=True),
|
||||
):
|
||||
moe_config = make_dummy_moe_config()
|
||||
# CPU's grouped-gemm kernels require hidden/intermediate sizes
|
||||
# aligned to 32; the size-1 defaults only work for backends that
|
||||
# don't check shapes at selection time.
|
||||
moe_config = (
|
||||
make_dummy_moe_config(hidden_dim=128, intermediate_size=128)
|
||||
if expected_backend == UnquantizedMoeBackend.CPU
|
||||
else make_dummy_moe_config()
|
||||
)
|
||||
selected_backend, expert_cls = select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
)
|
||||
|
||||
assert selected_backend == expected_backend
|
||||
if expected_backend in [
|
||||
UnquantizedMoeBackend.CPU,
|
||||
UnquantizedMoeBackend.OOT,
|
||||
UnquantizedMoeBackend.TPU,
|
||||
]:
|
||||
|
||||
Reference in New Issue
Block a user