[Model] Add DeepSeek-V4 CPU backend (#55355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-10 14:14:48 +08:00
committed by GitHub
parent 49632fb3ac
commit c3ccc0e957
47 changed files with 9564 additions and 627 deletions
+20 -14
View File
@@ -26,8 +26,8 @@ from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUUnquantizedExperts,
PowerCPUUnquantizedExperts,
X86CPUUnquantizedExperts,
select_experts,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -284,7 +284,9 @@ def test_cpu_fused_moe(
packed_w13 = cpu_prepack_moe_weight(w13, isa)
packed_w2 = cpu_prepack_moe_weight(w2, isa)
output = cpu_fused_moe(
output = torch.empty_like(input)
cpu_fused_moe(
output,
input,
packed_w13,
packed_w2,
@@ -368,7 +370,9 @@ def test_cpu_fused_moe_int8(
)
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
output = cpu_fused_moe_int8(
output = torch.empty_like(input)
cpu_fused_moe_int8(
output,
input,
packed_w13,
packed_w2,
@@ -521,15 +525,10 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
0.5 * hidden_size**0.5
)
# Route with the same helper apply() uses internally, so the reference
# only differs from the kernel in how the experts are evaluated.
topk_weight, topk_ids = select_experts(
hidden_states=input,
router_logits=router_logits,
top_k=topk_num,
use_grouped_topk=False,
renormalize=False,
)
# Route with the same helper the router uses internally, so the
# reference only differs from the kernel in how the experts are
# evaluated.
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
ref_output = ref_fused_moe(
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
@@ -575,15 +574,22 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
assert experts.isa == isa
experts.process_weights_after_loading(layer)
output = experts.apply(
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
topk_weights=topk_weight,
topk_ids=topk_ids,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
+48 -29
View File
@@ -197,14 +197,15 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
# Test inplace=False against reference
out = ops.fused_experts_cpu(
# Test writing into a fresh output buffer against reference
out = torch.empty_like(a)
ops.fused_experts_cpu(
out,
a.clone(),
pw1,
pw2,
topk_weight,
topk_ids,
False,
ops.CPUQuantMethod.FP8_W8A16,
w1_s,
w2_s,
@@ -215,14 +216,16 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
)
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
# Test inplace=True produces identical output
out_inplace = ops.fused_experts_cpu(
a.clone(),
# Test writing into the input tensor itself (aliased in-place) produces
# an identical result
a_inplace = a.clone()
ops.fused_experts_cpu(
a_inplace,
a_inplace,
pw1,
pw2,
topk_weight,
topk_ids,
True,
ops.CPUQuantMethod.FP8_W8A16,
w1_s,
w2_s,
@@ -231,7 +234,7 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
BLOCK_SIZE,
is_vnni=True,
)
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
torch.testing.assert_close(a_inplace, out, atol=0, rtol=0)
def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks():
@@ -249,13 +252,14 @@ def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks():
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE
)
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
out = ops.fused_experts_cpu(
out = torch.empty_like(a)
ops.fused_experts_cpu(
out,
a,
pw1,
pw2,
topk_weight,
topk_ids,
False,
ops.CPUQuantMethod.FP8_W8A16,
w1_s,
w2_s,
@@ -469,13 +473,15 @@ def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed):
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
# Kernel
out = ops.fused_experts_cpu(
a.clone(),
a_in = a.clone()
out = torch.empty_like(a_in)
ops.fused_experts_cpu(
out,
a_in,
pw1,
pw2,
topk_weight,
topk_ids,
False, # inplace
ops.CPUQuantMethod.MXFP4,
pw1s, # w1_scale
pw2s, # w2_scale
@@ -511,13 +517,14 @@ def test_mxfp4_cpu_fused_moe_small_expert_blocks():
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
out = ops.fused_experts_cpu(
out = torch.empty_like(a)
ops.fused_experts_cpu(
out,
a,
pw1,
pw2,
topk_weight,
topk_ids,
False,
ops.CPUQuantMethod.MXFP4,
pw1s,
pw2s,
@@ -569,13 +576,14 @@ def test_mxfp4_cpu_zero_codes_stay_zero(zero_byte, e8m0):
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
out = ops.fused_experts_cpu(
out = torch.empty_like(a)
ops.fused_experts_cpu(
out,
a,
pw1,
pw2,
topk_weight,
topk_ids,
False,
ops.CPUQuantMethod.MXFP4,
pw1s,
pw2s,
@@ -630,13 +638,14 @@ def test_mxfp4_cpu_zero_codes_mixed_with_nonzero(e8m0):
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
out = ops.fused_experts_cpu(
out = torch.empty_like(a)
ops.fused_experts_cpu(
out,
a,
pw1,
pw2,
topk_weight,
topk_ids,
False,
ops.CPUQuantMethod.MXFP4,
pw1s,
pw2s,
@@ -689,13 +698,15 @@ def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
# Kernel
out = ops.fused_experts_cpu(
a.clone(),
a_in = a.clone()
out = torch.empty_like(a_in)
ops.fused_experts_cpu(
out,
a_in,
pw1,
pw2,
topk_weight,
topk_ids,
False, # inplace
ops.CPUQuantMethod.MXFP4,
pw1s, # w1_scale
pw2s, # w2_scale
@@ -944,13 +955,15 @@ def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed
)
)
out = ops.fused_experts_cpu(
a.clone(),
a_in = a.clone()
out = torch.empty_like(a_in)
ops.fused_experts_cpu(
out,
a_in,
blocked_w1,
blocked_w2,
topk_weight,
topk_ids,
False, # inplace
ops.CPUQuantMethod.INT4_W4A8,
blocked_s1,
blocked_s2,
@@ -1051,7 +1064,11 @@ INT8_MOE_CONFIGS = [
@pytest.mark.parametrize("is_vnni", [False, True])
@pytest.mark.parametrize("inplace", [False, True])
def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
"""Test fused_experts_cpu INT8 W8A8 against torch reference."""
"""Test fused_experts_cpu INT8 W8A8 against torch reference.
``inplace`` exercises writing into a fresh output buffer vs. aliasing
the input tensor as the output buffer.
"""
set_random_seed(seed)
a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
@@ -1067,13 +1084,15 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
w1 = _prepack_experts(w1_q) if is_vnni else w1_q
w2 = _prepack_experts(w2_q) if is_vnni else w2_q
out = ops.fused_experts_cpu(
a.clone(),
a_in = a.clone()
out = a_in if inplace else torch.empty_like(a_in)
ops.fused_experts_cpu(
out,
a_in,
w1,
w2,
topk_weight,
topk_ids,
inplace,
ops.CPUQuantMethod.INT8_W8A8,
w1_s,
w2_s,
+30 -40
View File
@@ -2,8 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ZenTorch CPU fused MoE dispatch and forward."""
from types import SimpleNamespace
import pytest
import torch
@@ -27,8 +25,8 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
CPUUnquantizedExperts,
select_experts,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -95,18 +93,19 @@ def _make_layer_and_config(
@pytest.mark.parametrize("act", ZENTORCH_ACT)
def test_zen_cpu_fused_moe_dispatches_to_zentorch(act: MoEActivation):
"""When zentorch MoE is supported, experts select the zentorch path."""
layer, moe_config = _make_layer_and_config(act)
w13_before = layer.w13_weight.detach().clone()
w2_before = layer.w2_weight.detach().clone()
def test_zen_cpu_fused_moe_config_supported(act: MoEActivation):
"""CPUUnquantizedExperts is selected when zentorch MoE is supported."""
_, moe_config = _make_layer_and_config(act)
assert is_zentorch_moe_config_supported(moe_config)
experts = CPUUnquantizedExperts(moe_config, FusedMoEQuantConfig.make())
experts.process_weights_after_loading(layer)
assert experts._use_zentorch is True
torch.testing.assert_close(layer.w13_weight, w13_before)
torch.testing.assert_close(layer.w2_weight, w2_before)
supported, reason = CPUUnquantizedExperts.is_supported_config(
CPUUnquantizedExperts,
moe_config,
None,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
def test_zen_cpu_fused_moe_config_supported_unaligned_intermediate():
@@ -169,26 +168,34 @@ def test_zen_cpu_fused_moe_forward(
experts.process_weights_after_loading(layer)
assert experts._use_zentorch is True
topk_weight, topk_ids = select_experts(
hidden_states=input,
router_logits=router_logits,
top_k=topk_num,
use_grouped_topk=False,
renormalize=False,
)
w13_before = layer.w13_weight.detach().clone()
w2_before = layer.w2_weight.detach().clone()
output = experts.apply(
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=layer.w13_weight,
w2=layer.w2_weight,
router_logits=router_logits,
topk_weights=topk_weight,
topk_ids=topk_ids,
activation=act,
global_num_experts=expert_num,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
# zentorch needs no prepacking; weights are consumed as loaded.
torch.testing.assert_close(layer.w13_weight, w13_before)
torch.testing.assert_close(layer.w2_weight, w2_before)
ref_output = ref_fused_moe(
input,
layer.w13_weight,
@@ -202,20 +209,3 @@ def test_zen_cpu_fused_moe_forward(
atol, rtol = get_default_atol(output), get_default_rtol(output)
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
def test_zen_cpu_fused_moe_skips_without_activation():
"""Layers without an activation attribute fall back from zentorch."""
layer = SimpleNamespace()
moe_config = _make_moe_config(
expert_num=8,
hidden_size=128,
intermediate_size=128,
topk_num=4,
dtype=torch.bfloat16,
act=MoEActivation.SILU,
)
experts = CPUUnquantizedExperts(moe_config, FusedMoEQuantConfig.make())
experts.process_weights_after_loading(layer)
assert experts._use_zentorch is False
+12 -14
View File
@@ -18,14 +18,12 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
int8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
ZenCPUExpertsInt8,
select_experts,
)
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ZenCPUExpertsInt8
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
Int8MoeBackend,
backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
kInt8StaticChannelSym,
@@ -199,19 +197,27 @@ def test_zen_int8_dispatch_contract(
0.5 * HIDDEN_SIZE**0.5
)
router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16)
topk_weights, topk_ids = _softmax_topk(router_logits, topk_num, False)
experts = ZenCPUExpertsInt8(moe_config, quant_config)
# zentorch_fused_moe takes no expert_map.
assert not experts.supports_expert_map()
output = experts.apply(
output = torch.empty_like(input)
experts.apply(
output=output,
hidden_states=input,
w1=w13,
w2=w2,
router_logits=router_logits,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=act,
global_num_experts=EXPERT_NUM,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=torch.empty(0),
workspace2=torch.empty(0),
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
@@ -230,14 +236,6 @@ def test_zen_int8_dispatch_contract(
else:
assert call["w13_bias"] is None and call["w2_bias"] is None
topk_weights, topk_ids = select_experts(
hidden_states=input,
router_logits=router_logits,
use_grouped_topk=False,
top_k=topk_num,
renormalize=False,
scoring_func="softmax",
)
ref = ref_fused_moe_int8(
input,
w13,
File diff suppressed because it is too large Load Diff
@@ -92,6 +92,11 @@ _XPU_EXCLUDED_MODEL_IDS = {
"thinkingmachines/Inkling-NVFP4",
}
_CPU_EXCLUDED_MODEL_IDS = {
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
}
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
for model_arch in model_arch_list:
@@ -113,6 +118,11 @@ def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
if current_platform.is_cpu():
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
return model_ids
@@ -56,7 +56,7 @@ from vllm.v1.attention.backends.mla.flashmla_sparse import (
FlashMLASparseMetadataBuilder,
triton_convert_req_index_to_global_index,
)
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadataBuilder
from vllm.v1.attention.backends.utils import (
split_decodes_and_prefills,
split_prefill_chunks,
@@ -1424,7 +1424,7 @@ def test_sparse_backend_prefill_correctness(
def test_split_indexer_prefill_chunks(
seq_lens, query_lens, workspace_size, max_logits_bytes, expected
):
out = split_indexer_prefill_chunks(
out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
seq_lens,
query_lens,
workspace_size,
@@ -1438,7 +1438,9 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
seq_lens = torch.tensor([1000, 50])
query_lens = torch.tensor([100, 5])
out = split_indexer_prefill_chunks(seq_lens, query_lens, 2000, 1000)
out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
seq_lens, query_lens, 2000, 1000
)
# max_logits_elems = 250, N=1000 -> max_q = 1 -> 100 query sub-chunks
expected = [(slice(0, 1), slice(i, i + 1)) for i in range(100)]
# req1: M=5, N=50 -> 250 elems fits budget