[Model] Add DeepSeek-V4 CPU backend (#55355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -26,8 +26,8 @@ from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUUnquantizedExperts,
|
||||
PowerCPUUnquantizedExperts,
|
||||
X86CPUUnquantizedExperts,
|
||||
select_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -284,7 +284,9 @@ def test_cpu_fused_moe(
|
||||
|
||||
packed_w13 = cpu_prepack_moe_weight(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight(w2, isa)
|
||||
output = cpu_fused_moe(
|
||||
output = torch.empty_like(input)
|
||||
cpu_fused_moe(
|
||||
output,
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
@@ -368,7 +370,9 @@ def test_cpu_fused_moe_int8(
|
||||
)
|
||||
packed_w13 = cpu_prepack_moe_weight_int8(w13, isa)
|
||||
packed_w2 = cpu_prepack_moe_weight_int8(w2, isa)
|
||||
output = cpu_fused_moe_int8(
|
||||
output = torch.empty_like(input)
|
||||
cpu_fused_moe_int8(
|
||||
output,
|
||||
input,
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
@@ -521,15 +525,10 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
w2_bias = torch.randn((expert_num, hidden_size), dtype=dtype) / (
|
||||
0.5 * hidden_size**0.5
|
||||
)
|
||||
# Route with the same helper apply() uses internally, so the reference
|
||||
# only differs from the kernel in how the experts are evaluated.
|
||||
topk_weight, topk_ids = select_experts(
|
||||
hidden_states=input,
|
||||
router_logits=router_logits,
|
||||
top_k=topk_num,
|
||||
use_grouped_topk=False,
|
||||
renormalize=False,
|
||||
)
|
||||
# Route with the same helper the router uses internally, so the
|
||||
# reference only differs from the kernel in how the experts are
|
||||
# evaluated.
|
||||
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input, w13, w2, w13_bias, w2_bias, topk_weight, topk_ids, act
|
||||
@@ -575,15 +574,22 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
assert experts.isa == isa
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
output = experts.apply(
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
topk_weights=topk_weight,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=expert_num,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -197,14 +197,15 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
|
||||
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
|
||||
|
||||
# Test inplace=False against reference
|
||||
out = ops.fused_experts_cpu(
|
||||
# Test writing into a fresh output buffer against reference
|
||||
out = torch.empty_like(a)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a.clone(),
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
@@ -215,14 +216,16 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
)
|
||||
torch.testing.assert_close(ref_out.bfloat16(), out, atol=1e-2, rtol=1e-2)
|
||||
|
||||
# Test inplace=True produces identical output
|
||||
out_inplace = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
# Test writing into the input tensor itself (aliased in-place) produces
|
||||
# an identical result
|
||||
a_inplace = a.clone()
|
||||
ops.fused_experts_cpu(
|
||||
a_inplace,
|
||||
a_inplace,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
True,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
@@ -231,7 +234,7 @@ def test_w8a16_block_fp8_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
BLOCK_SIZE,
|
||||
is_vnni=True,
|
||||
)
|
||||
torch.testing.assert_close(out_inplace, out, atol=0, rtol=0)
|
||||
torch.testing.assert_close(a_inplace, out, atol=0, rtol=0)
|
||||
|
||||
|
||||
def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks():
|
||||
@@ -249,13 +252,14 @@ def test_w8a16_block_fp8_cpu_fused_moe_small_expert_blocks():
|
||||
a, w1, w2, w1_s, w2_s, topk_weight, topk_ids, BLOCK_SIZE
|
||||
)
|
||||
pw1, pw2 = _prepack_experts(w1), _prepack_experts(w2)
|
||||
out = ops.fused_experts_cpu(
|
||||
out = torch.empty_like(a)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.FP8_W8A16,
|
||||
w1_s,
|
||||
w2_s,
|
||||
@@ -469,13 +473,15 @@ def test_mxfp4_cpu_fused_moe(M, N, K, E, topk, seed):
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
|
||||
# Kernel
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
a_in = a.clone()
|
||||
out = torch.empty_like(a_in)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a_in,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s, # w1_scale
|
||||
pw2s, # w2_scale
|
||||
@@ -511,13 +517,14 @@ def test_mxfp4_cpu_fused_moe_small_expert_blocks():
|
||||
|
||||
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
out = ops.fused_experts_cpu(
|
||||
out = torch.empty_like(a)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s,
|
||||
pw2s,
|
||||
@@ -569,13 +576,14 @@ def test_mxfp4_cpu_zero_codes_stay_zero(zero_byte, e8m0):
|
||||
|
||||
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
out = ops.fused_experts_cpu(
|
||||
out = torch.empty_like(a)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s,
|
||||
pw2s,
|
||||
@@ -630,13 +638,14 @@ def test_mxfp4_cpu_zero_codes_mixed_with_nonzero(e8m0):
|
||||
|
||||
pw1, pw1s = _prepack_mxfp4_experts(w1q, w1s)
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
out = ops.fused_experts_cpu(
|
||||
out = torch.empty_like(a)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False,
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s,
|
||||
pw2s,
|
||||
@@ -689,13 +698,15 @@ def test_mxfp4_cpu_fused_moe_bias_swiglu(M, N, K, E, topk, seed):
|
||||
pw2, pw2s = _prepack_mxfp4_experts(w2q, w2s)
|
||||
|
||||
# Kernel
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
a_in = a.clone()
|
||||
out = torch.empty_like(a_in)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a_in,
|
||||
pw1,
|
||||
pw2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
ops.CPUQuantMethod.MXFP4,
|
||||
pw1s, # w1_scale
|
||||
pw2s, # w2_scale
|
||||
@@ -944,13 +955,15 @@ def test_int4_w4a16_cpu_fused_moe(M, N, K, E, topk, group_size, quant_algo, seed
|
||||
)
|
||||
)
|
||||
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
a_in = a.clone()
|
||||
out = torch.empty_like(a_in)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a_in,
|
||||
blocked_w1,
|
||||
blocked_w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
ops.CPUQuantMethod.INT4_W4A8,
|
||||
blocked_s1,
|
||||
blocked_s2,
|
||||
@@ -1051,7 +1064,11 @@ INT8_MOE_CONFIGS = [
|
||||
@pytest.mark.parametrize("is_vnni", [False, True])
|
||||
@pytest.mark.parametrize("inplace", [False, True])
|
||||
def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
|
||||
"""Test fused_experts_cpu INT8 W8A8 against torch reference."""
|
||||
"""Test fused_experts_cpu INT8 W8A8 against torch reference.
|
||||
|
||||
``inplace`` exercises writing into a fresh output buffer vs. aliasing
|
||||
the input tensor as the output buffer.
|
||||
"""
|
||||
set_random_seed(seed)
|
||||
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16) / (0.5 * K**0.5)
|
||||
@@ -1067,13 +1084,15 @@ def test_int8_w8a8_cpu_fused_moe(M, N, K, E, topk, seed, is_vnni, inplace):
|
||||
w1 = _prepack_experts(w1_q) if is_vnni else w1_q
|
||||
w2 = _prepack_experts(w2_q) if is_vnni else w2_q
|
||||
|
||||
out = ops.fused_experts_cpu(
|
||||
a.clone(),
|
||||
a_in = a.clone()
|
||||
out = a_in if inplace else torch.empty_like(a_in)
|
||||
ops.fused_experts_cpu(
|
||||
out,
|
||||
a_in,
|
||||
w1,
|
||||
w2,
|
||||
topk_weight,
|
||||
topk_ids,
|
||||
inplace,
|
||||
ops.CPUQuantMethod.INT8_W8A8,
|
||||
w1_s,
|
||||
w2_s,
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ZenTorch CPU fused MoE dispatch and forward."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -27,8 +25,8 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
CPUUnquantizedExperts,
|
||||
select_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -95,18 +93,19 @@ def _make_layer_and_config(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("act", ZENTORCH_ACT)
|
||||
def test_zen_cpu_fused_moe_dispatches_to_zentorch(act: MoEActivation):
|
||||
"""When zentorch MoE is supported, experts select the zentorch path."""
|
||||
layer, moe_config = _make_layer_and_config(act)
|
||||
w13_before = layer.w13_weight.detach().clone()
|
||||
w2_before = layer.w2_weight.detach().clone()
|
||||
def test_zen_cpu_fused_moe_config_supported(act: MoEActivation):
|
||||
"""CPUUnquantizedExperts is selected when zentorch MoE is supported."""
|
||||
_, moe_config = _make_layer_and_config(act)
|
||||
assert is_zentorch_moe_config_supported(moe_config)
|
||||
|
||||
experts = CPUUnquantizedExperts(moe_config, FusedMoEQuantConfig.make())
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
assert experts._use_zentorch is True
|
||||
torch.testing.assert_close(layer.w13_weight, w13_before)
|
||||
torch.testing.assert_close(layer.w2_weight, w2_before)
|
||||
supported, reason = CPUUnquantizedExperts.is_supported_config(
|
||||
CPUUnquantizedExperts,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
|
||||
def test_zen_cpu_fused_moe_config_supported_unaligned_intermediate():
|
||||
@@ -169,26 +168,34 @@ def test_zen_cpu_fused_moe_forward(
|
||||
experts.process_weights_after_loading(layer)
|
||||
assert experts._use_zentorch is True
|
||||
|
||||
topk_weight, topk_ids = select_experts(
|
||||
hidden_states=input,
|
||||
router_logits=router_logits,
|
||||
top_k=topk_num,
|
||||
use_grouped_topk=False,
|
||||
renormalize=False,
|
||||
)
|
||||
w13_before = layer.w13_weight.detach().clone()
|
||||
w2_before = layer.w2_weight.detach().clone()
|
||||
|
||||
output = experts.apply(
|
||||
topk_weight, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
router_logits=router_logits,
|
||||
topk_weights=topk_weight,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=expert_num,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
# zentorch needs no prepacking; weights are consumed as loaded.
|
||||
torch.testing.assert_close(layer.w13_weight, w13_before)
|
||||
torch.testing.assert_close(layer.w2_weight, w2_before)
|
||||
|
||||
ref_output = ref_fused_moe(
|
||||
input,
|
||||
layer.w13_weight,
|
||||
@@ -202,20 +209,3 @@ def test_zen_cpu_fused_moe_forward(
|
||||
|
||||
atol, rtol = get_default_atol(output), get_default_rtol(output)
|
||||
torch.testing.assert_close(output, ref_output, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_zen_cpu_fused_moe_skips_without_activation():
|
||||
"""Layers without an activation attribute fall back from zentorch."""
|
||||
layer = SimpleNamespace()
|
||||
moe_config = _make_moe_config(
|
||||
expert_num=8,
|
||||
hidden_size=128,
|
||||
intermediate_size=128,
|
||||
topk_num=4,
|
||||
dtype=torch.bfloat16,
|
||||
act=MoEActivation.SILU,
|
||||
)
|
||||
experts = CPUUnquantizedExperts(moe_config, FusedMoEQuantConfig.make())
|
||||
experts.process_weights_after_loading(layer)
|
||||
|
||||
assert experts._use_zentorch is False
|
||||
|
||||
@@ -18,14 +18,12 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
int8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
ZenCPUExpertsInt8,
|
||||
select_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ZenCPUExpertsInt8
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int8 import (
|
||||
Int8MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import _softmax_topk
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
kInt8StaticChannelSym,
|
||||
@@ -199,19 +197,27 @@ def test_zen_int8_dispatch_contract(
|
||||
0.5 * HIDDEN_SIZE**0.5
|
||||
)
|
||||
router_logits = torch.randn((BATCH_SIZE, EXPERT_NUM), dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids = _softmax_topk(router_logits, topk_num, False)
|
||||
|
||||
experts = ZenCPUExpertsInt8(moe_config, quant_config)
|
||||
# zentorch_fused_moe takes no expert_map.
|
||||
assert not experts.supports_expert_map()
|
||||
output = experts.apply(
|
||||
output = torch.empty_like(input)
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=input,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
router_logits=router_logits,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=act,
|
||||
global_num_experts=EXPERT_NUM,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=torch.empty(0),
|
||||
workspace2=torch.empty(0),
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
@@ -230,14 +236,6 @@ def test_zen_int8_dispatch_contract(
|
||||
else:
|
||||
assert call["w13_bias"] is None and call["w2_bias"] is None
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=input,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=False,
|
||||
top_k=topk_num,
|
||||
renormalize=False,
|
||||
scoring_func="softmax",
|
||||
)
|
||||
ref = ref_fused_moe_int8(
|
||||
input,
|
||||
w13,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -92,6 +92,11 @@ _XPU_EXCLUDED_MODEL_IDS = {
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
}
|
||||
|
||||
_CPU_EXCLUDED_MODEL_IDS = {
|
||||
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
|
||||
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
|
||||
}
|
||||
|
||||
|
||||
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
for model_arch in model_arch_list:
|
||||
@@ -113,6 +118,11 @@ def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
return model_ids
|
||||
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
FlashMLASparseMetadataBuilder,
|
||||
triton_convert_req_index_to_global_index,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import split_indexer_prefill_chunks
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
split_decodes_and_prefills,
|
||||
split_prefill_chunks,
|
||||
@@ -1424,7 +1424,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
def test_split_indexer_prefill_chunks(
|
||||
seq_lens, query_lens, workspace_size, max_logits_bytes, expected
|
||||
):
|
||||
out = split_indexer_prefill_chunks(
|
||||
out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
|
||||
seq_lens,
|
||||
query_lens,
|
||||
workspace_size,
|
||||
@@ -1438,7 +1438,9 @@ def test_split_indexer_prefill_chunks_single_request_overflow():
|
||||
seq_lens = torch.tensor([1000, 50])
|
||||
query_lens = torch.tensor([100, 5])
|
||||
|
||||
out = split_indexer_prefill_chunks(seq_lens, query_lens, 2000, 1000)
|
||||
out = DeepseekV32IndexerMetadataBuilder._split_indexer_prefill_chunks(
|
||||
seq_lens, query_lens, 2000, 1000
|
||||
)
|
||||
# max_logits_elems = 250, N=1000 -> max_q = 1 -> 100 query sub-chunks
|
||||
expected = [(slice(0, 1), slice(i, i + 1)) for i in range(100)]
|
||||
# req1: M=5, N=50 -> 250 elems fits budget
|
||||
|
||||
Reference in New Issue
Block a user