[Model] Add DeepSeek-V4 CPU backend (#55355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
Li, Jiang
2026-09-10 14:14:48 +08:00
committed by GitHub
parent 49632fb3ac
commit c3ccc0e957
47 changed files with 9564 additions and 627 deletions
@@ -6,6 +6,9 @@ import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils import replace_parameter
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
convert_to_channelwise,
)
@@ -281,10 +284,12 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
else params.weight_scale
)
assert weight_scale is not None
if weight_scale.dtype in (torch.float8_e8m0fnu, torch.uint8):
weight_scale = _upcast_e8m0_to_fp32(weight_scale.data)
replace_parameter(
layer,
scale_attr,
torch.nn.Parameter(weight_scale.data, requires_grad=False),
torch.nn.Parameter(weight_scale.data.contiguous(), requires_grad=False),
)
def apply_weights(
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .aiter import *
from .cpu import *
from .tilelang import *
from .torch import *
from .triton import *
@@ -14,6 +15,9 @@ __all__ = [
"mhc_post_aiter",
"mhc_fused_post_pre_aiter",
"hc_head_fused_aiter",
"mhc_pre_cpu",
"mhc_post_cpu",
"hc_head_fused_cpu",
"mhc_pre_tilelang",
"mhc_post_tilelang",
"mhc_fused_post_pre_tilelang",
+96
View File
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm import _custom_ops as ops
def mhc_pre_cpu(
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
multiplier to 2.0 -- true of every real call site in
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
"""
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
assert hc_pre_eps == hc_sinkhorn_eps, (
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
)
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = residual.reshape(-1, hc_mult, hidden_size)
layer_input, post, comb = ops.hc_pre_fused_cpu(
x_flat,
fn,
hc_scale,
hc_base,
hc_mult,
sinkhorn_repeat,
rms_eps,
hc_pre_eps,
)
post_mix = post.view(*outer_shape, hc_mult, 1)
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
layer_input = layer_input.view(*outer_shape, hidden_size)
return post_mix, comb_mix, layer_input
def mhc_post_cpu(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
hc_mult, hidden_size = residual.shape[-2:]
outer_shape = residual.shape[:-2]
x_flat = x.reshape(-1, hidden_size)
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
return out.view(*outer_shape, hc_mult, hidden_size)
def hc_head_fused_cpu(
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
tests/kernels/test_mhc_kernels.py for the eager reference this is tested
against).
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
the real call site in `models/deepseek_v4/cpu/model.py`.
"""
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
return out.view(*outer_shape, hidden_size)
@@ -218,7 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
Returns:
Output tensor after MoE computation
"""
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
from vllm.model_executor.layers.fused_moe.router.cpu_router import (
select_experts,
)
@@ -4,7 +4,6 @@
import math
import sys
from collections.abc import Callable
from typing import cast
import torch
@@ -34,6 +33,9 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
@@ -48,133 +50,26 @@ from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.math_utils import round_up
logger = init_logger(__name__)
# ===========================================================================
# Routing
# ===========================================================================
def grouped_topk(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_expert_group: int = 0,
topk_group: int = 0,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
gating_output = gating_output.float()
if scoring_func == "softmax":
scores = torch.softmax(gating_output, dim=-1)
elif scoring_func == "sigmoid":
scores = gating_output.sigmoid()
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
num_token = scores.shape[0]
if e_score_correction_bias is not None:
original_scores = scores
scores = scores + e_score_correction_bias.unsqueeze(0)
group_scores = (
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
)
else:
group_scores = (
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
) # [n, n_group]
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
1
] # [n, top_k_group]
group_mask = torch.zeros_like(group_scores) # [n, n_group]
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
.reshape(num_token, -1)
) # [n, e]
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
if e_score_correction_bias is not None:
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
topk_weights = original_scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights, topk_ids.to(torch.int32)
def select_experts(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
top_k: int,
use_grouped_topk: bool,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if use_grouped_topk:
assert topk_group is not None
assert num_expert_group is not None
return grouped_topk(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
num_expert_group=num_expert_group,
topk_group=topk_group,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
elif custom_routing_function is None:
assert scoring_func == "softmax"
topk_logit_vals, topk_idx = torch.topk(
router_logits, k=top_k, dim=-1, sorted=False
)
if renormalize:
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
else:
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
topk_vals = (topk_logit_vals - logZ).exp()
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
else:
topk_weights, topk_ids = custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
topk=top_k,
renormalize=renormalize,
)
# cpu_fused_moe reads routing tensors as contiguous float32/int32
# buffers and does not account for tensor strides.
topk_weights = topk_weights.to(torch.float32).contiguous()
topk_ids = topk_ids.to(torch.int32).contiguous()
return topk_weights, topk_ids
# ===========================================================================
# Unquantized (BF16/FP16/FP32) MoE
# ===========================================================================
class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
"""Portable vector grouped-gemm unquantized MoE experts."""
isa = "vec"
output_alignment = 32
reduction_alignment = 1
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self._use_zentorch = False
@classmethod
def _intermediate_alignment(cls) -> int:
return math.lcm(cls.output_alignment, cls.reduction_alignment)
@@ -210,20 +105,6 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
)
return True, None
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
# Router configuration that the monolithic apply() signature cannot
# carry. Captured off the layer in process_weights_after_loading.
self.use_grouped_topk = False
self.renormalize = False
self.scoring_func = "softmax"
self.custom_routing_function: Callable | None = None
self._use_zentorch = False
@property
def expects_unquantized_inputs(self) -> bool:
return True
@@ -286,8 +167,8 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
# Routing runs in select_experts(), which covers every routing method
# a layer can be configured with, including custom routing functions.
# Routing runs in the CPURouter, which covers every routing method a
# layer can be configured with, including custom routing functions.
return True
@staticmethod
@@ -298,10 +179,6 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
return True
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.use_grouped_topk = layer.use_grouped_topk
self.renormalize = layer.renormalize
self.scoring_func = layer.scoring_func
self.custom_routing_function = layer.custom_routing_function
self._use_zentorch = is_zentorch_moe_supported(layer)
if self._use_zentorch:
return
@@ -345,65 +222,44 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
# this very object, which is what feeds self.w1_bias in apply().
layer.w13_bias.data = new_bias
def _select_experts(
def workspace_shapes(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
num_expert_group: int | None,
topk_group: int | None,
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float | None,
) -> tuple[torch.Tensor, torch.Tensor]:
return select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
top_k=self.moe_config.experts_per_token,
use_grouped_topk=self.use_grouped_topk,
renormalize=self.renormalize,
topk_group=topk_group,
num_expert_group=num_expert_group,
custom_routing_function=self.custom_routing_function,
scoring_func=self.scoring_func,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# cpu_fused_moe manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = self._select_experts(
hidden_states,
router_logits,
num_expert_group,
topk_group,
e_score_correction_bias,
routed_scaling_factor,
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
) -> None:
# apply_router_weight_on_input (topk=1 only) is already applied to
# hidden_states by MoEPrepareAndFinalizeNoDPEPModular.prepare().
if self._use_zentorch:
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
@@ -416,9 +272,9 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
apply_router_weight_on_input,
str(activation.value).lower(),
)
return output
return cpu_fused_moe(
return
cpu_fused_moe(
output,
hidden_states,
w1,
w2,
@@ -460,8 +316,6 @@ class X86CPUUnquantizedExperts(CPUUnquantizedExperts):
)
if not supported:
return supported, reason
if is_zentorch_moe_config_supported(moe_config):
return True, None
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
@@ -496,8 +350,6 @@ class ArmCPUUnquantizedExperts(CPUUnquantizedExperts):
)
if not supported:
return supported, reason
if is_zentorch_moe_config_supported(moe_config):
return True, None
if moe_config.in_dtype != torch.bfloat16:
return False, "kernel requires bfloat16 activations"
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
@@ -552,8 +404,8 @@ def prepare_fp8_moe_layer_for_cpu(
return packed_w13, packed_w2
class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
"""CPU FP8 W8A16 block-quantized monolithic MoE experts."""
class CPUExpertsFp8(mk.FusedMoEExpertsModular):
"""CPU FP8 W8A16 block-quantized modular MoE experts."""
def __init__(
self,
@@ -615,6 +467,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
@@ -624,42 +477,49 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
) -> bool:
return True
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# fused_experts_cpu manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
) -> None:
if apply_router_weight_on_input:
# fused_experts_cpu always applies topk_weights internally on
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
# also pre-apply it to hidden_states, double-weighting the
# output. Not needed by any CPU FP8 model today.
raise NotImplementedError(
"CPUExpertsFp8 does not support apply_router_weight_on_input=True."
)
block_shape = (
list(self.quant_config.block_shape)
if self.quant_config.block_shape
@@ -670,13 +530,13 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
)
)
return fused_experts_cpu(
fused_experts_cpu(
output,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.FP8_W8A16, # moe_comp_method
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
@@ -710,8 +570,8 @@ def prepare_mxfp4_moe_layer_for_cpu(
return packed_w13, packed_w2, packed_w13_scale, packed_w2_scale
class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
"""CPU MXFP4 W4A16 monolithic MoE experts."""
class CPUExpertsMxfp4(mk.FusedMoEExpertsModular):
"""CPU MXFP4 W4A16 modular MoE experts."""
def __init__(
self,
@@ -773,6 +633,7 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.Default,
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
]
@staticmethod
@@ -782,55 +643,62 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
) -> bool:
return True
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# fused_experts_cpu manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
) -> None:
if apply_router_weight_on_input:
# fused_experts_cpu always applies topk_weights internally on
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
# also pre-apply it to hidden_states, double-weighting the
# output. Not needed by any CPU MXFP4 model today.
raise NotImplementedError(
"CPUExpertsMxfp4 does not support apply_router_weight_on_input=True."
)
# Get bias and swiglu params from quant config
w1_bias = self.quant_config.w1_bias
w2_bias = self.quant_config.w2_bias
alpha = getattr(self.quant_config, "gemm1_alpha", None)
limit = getattr(self.quant_config, "gemm1_clamp_limit", None)
return fused_experts_cpu(
fused_experts_cpu(
output,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.MXFP4, # moe_comp_method
self.w1_scale, # w1_scale
self.w2_scale, # w2_scale
@@ -911,8 +779,8 @@ def prepare_int4_moe_layer_for_cpu(
return (blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
"""CPU INT4 W4A16 group-quantized monolithic MoE experts.
class CPUExpertsInt4(mk.FusedMoEExpertsModular):
"""CPU INT4 W4A16 group-quantized modular MoE experts.
Weights are int4 (packed), activations are bf16/fp16.
Internally uses int8 compute via fused_experts_cpu with INT4_W4A8.
@@ -992,55 +860,75 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
def supports_expert_map(self) -> bool:
return False
def moe_problem_size(
self,
a1: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
# convert_weight_packed_scale_zp blocks w1/w2 into AMX tiles
# (E, Nc, Kc, buffer_bytes), not the (E, N, K) layout the base
# implementation assumes -- N/K aren't recoverable from that shape,
# so read them from moe_config instead.
E = w1.shape[0]
K = a1.size(-1)
N = (
self.moe_config.intermediate_size_per_partition
* self.moe_config.w13_num_shards
)
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
topk = topk_ids.size(1)
return E, M, N, K, topk
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# fused_experts_cpu manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
) -> None:
if apply_router_weight_on_input:
raise NotImplementedError(
"CPUExpertsInt4 (W4A16) does not support "
"apply_router_weight_on_input=True. "
)
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
fused_experts_cpu(
output,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.INT4_W4A8,
self.w1_scale,
self.w2_scale,
@@ -1060,9 +948,9 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
# ===========================================================================
class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
monolithic MoE experts."""
modular MoE experts."""
def __init__(
self,
@@ -1172,49 +1060,56 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# fused_experts_cpu manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
# grouped topk + fused topk bias parameters
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
return fused_experts_cpu(
) -> None:
if apply_router_weight_on_input:
# fused_experts_cpu always applies topk_weights internally on
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
# also pre-apply it to hidden_states, double-weighting the
# output. Not needed by any CPU INT8 W8A8 model today.
raise NotImplementedError(
"CPUExpertsInt8 does not support apply_router_weight_on_input=True."
)
fused_experts_cpu(
output,
hidden_states,
w1,
w2,
topk_weights,
topk_ids,
False, # inplace
CPUQuantMethod.INT8_W8A8,
self.w1_scale,
self.w2_scale,
@@ -1229,7 +1124,7 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
)
class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
"""Arm INT8 MoE with per-token activation and channelwise weight quantization."""
@property
@@ -1331,48 +1226,47 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# cpu_fused_moe_int8 manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
) -> None:
# apply_router_weight_on_input (topk=1 only) is already applied to
# hidden_states by MoEPrepareAndFinalizeNoDPEPModular.prepare().
assert self.w1_scale is not None
assert self.w2_scale is not None
return cpu_fused_moe_int8(
cpu_fused_moe_int8(
output,
hidden_states,
w1,
w2,
@@ -1388,7 +1282,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
)
class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
quantization, dispatched through zentorch."""
@@ -1424,9 +1318,7 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
if self.w2_bias is None
else self.w2_bias.detach().to(torch.bfloat16).contiguous()
)
logger.info_once(
"[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE (monolithic experts)"
)
logger.info_once("[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE")
@property
def expects_unquantized_inputs(self) -> bool:
@@ -1487,48 +1379,47 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
def supports_expert_map(self) -> bool:
return False
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# zentorch_fused_moe manages its own scratch space.
return (0,), (0,), (M, K)
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
topk_weights, topk_ids = select_experts(
hidden_states=hidden_states,
router_logits=router_logits,
use_grouped_topk=num_expert_group is not None,
top_k=self.moe_config.experts_per_token,
renormalize=self.moe_config.routing_method
in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
),
topk_group=topk_group,
num_expert_group=num_expert_group,
scoring_func="softmax",
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
e_score_correction_bias=e_score_correction_bias,
)
) -> None:
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input is only implemented for topk=1"
)
hidden_states = hidden_states.mul(topk_weights.to(hidden_states.dtype))
output = torch.empty_like(hidden_states)
torch.ops.zentorch.zentorch_fused_moe(
output,
hidden_states,
@@ -1543,4 +1434,3 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
self._w1_scale_bf16,
self._w2_scale_bf16,
)
return output
@@ -1175,6 +1175,24 @@ class FusedMoEKernelModularImpl:
# time we need cache3, we're done with cache1.
# Reuse workspace13 for the output since there is only one chunk.
max_shape_size = max(prod(workspace13_shape), prod(fused_out_shape))
if current_platform.is_cpu():
# Every CPU FusedMoEExpertsModular kernel reports zero-sized
# workspace13/workspace2 (it manages its own scratch space
# internally) and just needs the output buffer, so skip the
# workspace manager entirely here -- it's the one part of this
# call graph Dynamo can't trace (ContextVar-based lane lookup),
# and CPU never actually needs its cross-chunk memory reuse.
common_workspace = torch.empty(
(max_shape_size,), dtype=workspace_dtype, device=device
)
workspace2 = torch.empty(
workspace2_shape, dtype=workspace_dtype, device=device
)
workspace13 = _resize_cache(common_workspace, workspace13_shape)
fused_out = _resize_cache(common_workspace, fused_out_shape)
return workspace13, workspace2, fused_out
common_workspace, workspace2 = current_workspace_manager().get_simultaneous(
((max_shape_size,), workspace_dtype),
(workspace2_shape, workspace_dtype),
@@ -348,6 +348,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.MARLIN,
Mxfp4MoeBackend.BATCHED_MARLIN,
Mxfp4MoeBackend.XPU,
Mxfp4MoeBackend.CPU,
Mxfp4MoeBackend.EMULATION,
]
return _AVAILABLE_BACKENDS
@@ -366,6 +367,8 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
]
if current_platform.is_xpu():
return [Mxfp4MoeBackend.XPU]
if current_platform.is_cpu():
return [Mxfp4MoeBackend.CPU]
_AVAILABLE_BACKENDS = [
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
Mxfp4MoeBackend.DEEPGEMM_MXFP4,
@@ -627,17 +630,6 @@ def select_mxfp4_moe_backend(
activation_format,
)
if current_platform.is_cpu():
backend = Mxfp4MoeBackend.CPU
logger.info_once(_make_log_backend(backend))
return _return_or_raise(
Mxfp4MoeBackend.CPU,
config,
kMxfp4Static,
None,
activation_format,
)
unsupported_log = "; ".join(
[
f"backend: {backend.value}, reason: {reason}"
@@ -1813,11 +1805,36 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias=w2_bias,
_cache_permute_indices=_cache_permute_indices,
)
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_mxfp4_moe_layer_for_cpu,
)
packed_w13, packed_w2, packed_w13_scale, packed_w2_scale = (
prepare_mxfp4_moe_layer_for_cpu(
w13_weight.data,
w2_weight.data,
w13_weight_scale.data,
w2_weight_scale.data,
)
)
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
return (
packed_w13,
packed_w2,
packed_w13_scale,
packed_w2_scale,
w13_bias,
w2_bias,
)
else:
raise ValueError(
f"Unsupported mxfp4_backend for Mxfp4MoEMethod: {mxfp4_backend}. "
"Expected TRTLLM, FlashInfer CUTLASS, Triton, AITER, XPU, or "
"emulation backend."
"Expected TRTLLM, FlashInfer CUTLASS, Triton, AITER, XPU, "
"CPU, or emulation backend."
)
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Router for CPU MoE experts.
This is the routing logic that used to live inline in each CPU
FusedMoEExpertsMonolithic.apply() (see git history of
experts/cpu_moe.py); factoring it out into a proper FusedMoERouter is what
lets CPU MoE experts be FusedMoEExpertsModular like every other backend.
The math below is unchanged from that prior inline version -- this is a
relocation, not a rewrite.
"""
from collections.abc import Callable
import torch
from vllm._custom_ops import biased_topk_cpu, hash_topk_cpu
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
def _grouped_topk(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
num_expert_group: int,
topk_group: int,
scoring_func: str,
routed_scaling_factor: float,
e_score_correction_bias: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
gating_output = gating_output.float()
if scoring_func == "softmax":
scores = torch.softmax(gating_output, dim=-1)
elif scoring_func == "sigmoid":
scores = gating_output.sigmoid()
else:
raise ValueError(f"Unsupported scoring function: {scoring_func}")
num_token = scores.shape[0]
if e_score_correction_bias is not None:
original_scores = scores
scores = scores + e_score_correction_bias.unsqueeze(0)
group_scores = (
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
)
else:
group_scores = (
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
) # [n, n_group]
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
1
] # [n, top_k_group]
group_mask = torch.zeros_like(group_scores) # [n, n_group]
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
score_mask = (
group_mask.unsqueeze(-1)
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
.reshape(num_token, -1)
) # [n, e]
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
if e_score_correction_bias is not None:
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
topk_weights = original_scores.gather(1, topk_ids)
else:
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights, topk_ids.to(torch.int32)
def _sqrtsoftplus_bias_topk(
router_logits: torch.Tensor,
top_k: int,
renormalize: bool,
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float,
input_ids: torch.Tensor | None,
hash_indices_table: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""DeepSeek V4's routing: weight = sqrt(softplus(logit)); expert
selection uses a bias-corrected score (weight + correction_bias), but
the returned weight for each selected expert is the *unbiased* value
(the "noaux_tc" pattern -- bias steers selection only). Always
renormalizes selected weights to sum to ``routed_scaling_factor``,
matching the reference Triton kernel
(vllm/model_executor/layers/fused_moe/router/dsv4_topk.py).
When ``hash_indices_table`` is given (hash-routed layers), expert
selection instead comes from ``hash_indices_table[input_ids]`` --
the bias-corrected-score topk is skipped entirely, matching the CUDA
reference kernel (``dsv4HashTopkSoftplusSqrt``).
Backed by the ported SGLang AVX512 kernels `biased_topk_cpu` (the
correction-bias path) and `hash_topk_cpu` (the hash-routed-layer path);
see csrc/cpu/sgl-kernels/topk.cpp.
"""
assert renormalize, "DeepseekV4 routing always renormalizes"
gating_output = router_logits.contiguous()
num_experts = gating_output.shape[1]
if hash_indices_table is not None:
assert input_ids is not None
# hash_indices_table is int32 by construction (see cpu/model.py) and
# advanced indexing always allocates a fresh contiguous tensor.
tid2eid = hash_indices_table[input_ids]
return hash_topk_cpu(
gating_output,
tid2eid,
top_k,
"sqrtsoftplus",
0, # num_fused_shared_experts: never fused into routing on CPU
num_experts,
routed_scaling_factor,
)
assert e_score_correction_bias is not None
return biased_topk_cpu(
gating_output, # only used for num_tokens/device (dtype forced fp32)
gating_output,
e_score_correction_bias.contiguous(),
top_k,
renormalize,
"sqrtsoftplus",
0, # num_fused_shared_experts: never fused into routing on CPU
routed_scaling_factor,
True, # apply_routed_scaling_factor_on_output
)
def _softmax_topk(
router_logits: torch.Tensor,
top_k: int,
renormalize: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
topk_logit_vals, topk_idx = torch.topk(router_logits, k=top_k, dim=-1, sorted=False)
if renormalize:
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
else:
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
topk_vals = (topk_logit_vals - logZ).exp()
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
class CPURouter(BaseRouter):
"""Router covering every routing scheme CPU MoE experts support: plain
softmax top-k, grouped top-k (with an optional correction bias), an
arbitrary custom_routing_function, and DeepSeek V4's sqrtsoftplus scheme
(bias-corrected or hash-routed via the ported biased_topk_cpu/
hash_topk_cpu kernels). All of the above except sqrtsoftplus are plain
torch ops so Dynamo traces straight through into them like it did when
this logic lived inline in FusedMoEExpertsMonolithic.apply() -- no new
compile boundary is introduced here.
"""
def __init__(
self,
top_k: int,
global_num_experts: int,
use_grouped_topk: bool = False,
num_expert_group: int | None = None,
topk_group: int | None = None,
renormalize: bool = True,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
custom_routing_function: Callable | None = None,
hash_indices_table: torch.Tensor | None = None,
eplb_state: EplbLayerState | None = None,
):
super().__init__(
top_k=top_k,
global_num_experts=global_num_experts,
eplb_state=eplb_state,
)
self.use_grouped_topk = use_grouped_topk
self.num_expert_group = num_expert_group
self.topk_group = topk_group
self.renormalize = renormalize
self.scoring_func = scoring_func
self.routed_scaling_factor = routed_scaling_factor
self.e_score_correction_bias = e_score_correction_bias
self.custom_routing_function = custom_routing_function
self._hash_indices_table = hash_indices_table
@property
def routing_method_type(self) -> RoutingMethodType:
return get_routing_method_type(
scoring_func=self.scoring_func,
top_k=self.top_k,
renormalize=self.renormalize,
num_expert_group=self.num_expert_group,
has_e_score_bias=self.e_score_correction_bias is not None,
routed_scaling_factor=self.routed_scaling_factor,
)
def _compute_routing(
self,
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
indices_type: torch.dtype | None,
*,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.use_grouped_topk:
assert self.topk_group is not None
assert self.num_expert_group is not None
return _grouped_topk(
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
num_expert_group=self.num_expert_group,
topk_group=self.topk_group,
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
e_score_correction_bias=self.e_score_correction_bias,
)
elif self.scoring_func == "sqrtsoftplus":
assert (
self._hash_indices_table is not None
or self.e_score_correction_bias is not None
)
return _sqrtsoftplus_bias_topk(
router_logits=router_logits,
top_k=self.top_k,
renormalize=self.renormalize,
e_score_correction_bias=self.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
input_ids=input_ids,
hash_indices_table=self._hash_indices_table,
)
elif self.custom_routing_function is None:
assert self.scoring_func == "softmax"
return _softmax_topk(router_logits, self.top_k, self.renormalize)
else:
topk_weights, topk_ids = self.custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
topk=self.top_k,
renormalize=self.renormalize,
)
# cpu_fused_moe reads routing tensors as contiguous float32/int32
# buffers and does not account for tensor strides.
topk_weights = topk_weights.to(torch.float32).contiguous()
topk_ids = topk_ids.to(torch.int32).contiguous()
return topk_weights, topk_ids
def select_experts(
hidden_states: torch.Tensor,
router_logits: torch.Tensor,
top_k: int,
use_grouped_topk: bool,
renormalize: bool,
topk_group: int | None = None,
num_expert_group: int | None = None,
custom_routing_function: Callable | None = None,
scoring_func: str = "softmax",
routed_scaling_factor: float = 1.0,
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Routing helper for the CPU MoE experts that still use the monolithic
contract and so must compute routing themselves from ``router_logits``
(the zentorch-accelerated path and the ARM W4A8 dynamic-quant kernel,
kept monolithic since zentorch/Arm hardware isn't available to validate
a modular migration here). Modular CPU experts get routing from
``CPURouter`` instead; this is a thin wrapper around the same logic for
the two classes that can't use it.
"""
router = CPURouter(
top_k=top_k,
global_num_experts=router_logits.shape[-1],
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
renormalize=renormalize,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
custom_routing_function=custom_routing_function,
hash_indices_table=hash_indices_table,
)
return router._compute_routing(
hidden_states, router_logits, None, input_ids=input_ids
)
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501
AiterSharedRoutedFusedMoERouter,
)
from vllm.model_executor.layers.fused_moe.router.cpu_router import CPURouter
from vllm.model_executor.layers.fused_moe.router.custom_routing_router import (
CustomRoutingRouter,
)
@@ -35,6 +36,7 @@ from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
ZeroExpertRouter,
)
from vllm.platforms import current_platform
def create_fused_moe_router(
@@ -141,6 +143,26 @@ def create_fused_moe_router(
routed_scaling_factor=routed_scaling_factor,
)
if current_platform.is_cpu():
# CPURouter covers every routing scheme CPU MoE experts support
# (plain softmax, grouped topk, custom routing functions, and
# DeepSeek V4's sqrtsoftplus/hash routing) in a single class, so it
# takes priority over the scheme-specific routers below.
return CPURouter(
top_k=top_k,
global_num_experts=global_num_experts,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
renormalize=renormalize,
scoring_func=scoring_func,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
custom_routing_function=custom_routing_function,
hash_indices_table=hash_indices_table,
eplb_state=eplb_state,
)
if use_grouped_topk:
assert custom_routing_function is None
if num_expert_group is None or topk_group is None:
@@ -317,10 +317,17 @@ class MoERunner(MoERunnerInterface):
return self.routed_experts.load_weights(weights)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
if current_platform.is_tpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we
# will switch to using the moe_forward custom op.
# Note: CPU doesn't require wrapped _forward_impl.
return _moe_forward if self._shared_experts is None else _moe_forward_shared
if current_platform.is_cpu():
# CPU never touches the workspace manager (Monolithic experts
# skip it entirely; Modular experts' _allocate_buffers bypasses
# it too, see modular_kernel.py) -- the ContextVar-based lane
# lookup was the only part of this call graph Dynamo can't
# trace, so CPU can always call the fused-MoE op directly.
return _moe_forward if self._shared_experts is None else _moe_forward_shared
return (
+88
View File
@@ -261,6 +261,36 @@ class MHCPreOp(CustomOp):
sinkhorn_repeat,
)
def forward_cpu(
self,
residual: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return mhc_kernels.mhc_pre_cpu(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
n_splits,
norm_weight,
norm_eps,
)
# --8<-- [start:mhc_post]
@CustomOp.register("mhc_post")
@@ -337,6 +367,15 @@ class MHCPostOp(CustomOp):
comb_res_mix,
)
def forward_cpu(
self,
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
) -> torch.Tensor:
return mhc_kernels.mhc_post_cpu(x, residual, post_layer_mix, comb_res_mix)
# --8<-- [start:hc_head]
@CustomOp.register("hc_head")
@@ -444,6 +483,19 @@ class HCHeadOp(CustomOp):
)
return out.view(*outer_shape, hidden_size)
def forward_cpu(
self,
hidden_states: torch.Tensor,
hc_fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_norm_eps: float,
hc_eps: float,
) -> torch.Tensor:
return mhc_kernels.hc_head_fused_cpu(
hidden_states, hc_fn, hc_scale, hc_base, rms_norm_eps, hc_eps
)
# --8<-- [start:mhc_fused_post_pre]
@CustomOp.register("mhc_fused_post_pre")
@@ -654,6 +706,42 @@ class MHCFusedPostPreOp(CustomOp):
sinkhorn_repeat,
)
def forward_cpu(
self,
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
n_splits: int = 1,
tile_n: int = 1,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 0.0,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Decompose into post + pre (no fused kernel available).
residual_cur = mhc_kernels.mhc_post_cpu(
x, residual, post_layer_mix, comb_res_mix
)
post_mix_cur, comb_mix_cur, layer_input_cur = mhc_kernels.mhc_pre_cpu(
residual_cur,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
)
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
"""[s, hidden_size] -> [s, n * hidden_size] by replication."""
@@ -828,10 +828,12 @@ class SparseAttnIndexer(CustomOp):
return self.forward_cuda(hidden_states, q_quant, k, weights)
elif current_platform.is_rocm():
return self.forward_hip(hidden_states, q_quant, k, weights)
elif current_platform.is_cpu():
return self.forward_cpu(hidden_states, q_quant, k, weights)
else:
raise NotImplementedError(
"SparseAttnIndexer native forward is only implemented for "
"CUDA, ROCm and XPU platforms."
"CUDA, ROCm, XPU and CPU platforms."
)
def forward_cuda(
@@ -919,3 +921,196 @@ class SparseAttnIndexer(CustomOp):
"Sparse attention indexer ROCm path is only supported on AITER. "
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
)
def forward_cpu(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
):
"""CPU sparse attention indexer: cache write stays eager Python glue
(own K-cache layout, not shared with the main attention cache
write). PREFILL and DECODE both call the ported
``fp8_paged_mqa_logits_cpu``/``topk_transform_512_cpu`` kernels,
which read the paged K-cache directly via ``page_table`` -- no
eager gather step, no per-request Python loop.
``prefill_metadata.chunks`` always has exactly one entry here:
``DeepseekV4CPUIndexerMetadataBuilder`` overrides the base chunk
split to always return the whole step's prefill batch as one
chunk, since the base chunking only bounds CUDA/XPU's dense M*N
logits tensor and flat K-gather workspace, neither of which this
paged kernel allocates.
"""
assert not self.use_fp4_cache, (
"CPU sparse indexer doesn't support fp4 cache yet"
)
assert isinstance(q_quant, torch.Tensor), (
"CPU sparse_attn_indexer expects a single FP8 q_quant tensor"
)
assert self.dcp_world_size <= 1 and not self.use_pcp, (
"CPU sparse indexer doesn't support decode/prefill context parallelism yet."
)
forward_context = get_forward_context()
attn_metadata = forward_context.attn_metadata
attn_metadata_narrowed: DeepseekV32IndexerMetadata | None = None
if isinstance(attn_metadata, dict):
metadata = attn_metadata[self.k_cache.prefix]
assert isinstance(metadata, DeepseekV32IndexerMetadata)
attn_metadata_narrowed = metadata
if attn_metadata_narrowed is None:
# Profiling/dummy run: no real metadata to act on.
return self.topk_indices_buffer
kv_cache = self.k_cache.kv_cache
topk_tokens = self.topk_tokens
topk_indices_buffer = self.topk_indices_buffer
slot_mapping = attn_metadata_narrowed.slot_mapping
has_decode = attn_metadata_narrowed.num_decodes > 0
has_prefill = attn_metadata_narrowed.num_prefills > 0
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
num_tokens = slot_mapping.shape[0]
if k is not None:
k = k[:num_tokens]
if not self.skip_k_cache_insert:
# Only reachable via DeepseekV32Attention with
# prefill_context_parallel_size > 1 on CPU -- set_k_cpu/set_s_cpu
# (the kernels this used to call) have been removed as unused/
# untested (csrc/cpu/sgl-kernels/store_cache.cpp).
raise NotImplementedError(
"SparseAttnIndexer.forward_cpu: skip_k_cache_insert=False "
"(prefill context parallel on CPU) is not supported."
)
topk_indices_buffer[: hidden_states.shape[0]] = -1
if has_prefill:
assert topk_tokens == 512, (
"topk_transform_512_cpu only supports index_topk == 512."
)
prefill_metadata = attn_metadata_narrowed.prefill
assert prefill_metadata is not None
assert len(prefill_metadata.chunks) == 1, (
"forward_cpu expects the prefill metadata builder to always "
"produce a single chunk -- see "
"DeepseekV4CPUIndexerMetadataBuilder._split_indexer_prefill_chunks."
)
chunk = prefill_metadata.chunks[0]
# kv_cache is a per-layer view into vLLM's shared multi-layer
# cache allocation, so its block stride generally exceeds
# block_size * page_width; fp8_paged_mqa_logits_cpu reads
# kv_view.stride(0) explicitly, so no copy is needed here.
kv_view = kv_cache.view(kv_cache.shape[0], -1)
block_size = kv_cache.shape[1]
q_slice = q_quant[chunk.token_start : chunk.token_end]
topk_indices = topk_indices_buffer[
chunk.token_start : chunk.token_end, :topk_tokens
]
if chunk.local_total_seq_lens == 0:
topk_indices.fill_(-1)
else:
assert chunk.local_cu_seq_lens is not None
# Each token's own (per-request, DCP-local) causal length.
local_seq_lens = chunk.cu_seqlen_ke - chunk.cu_seqlen_ks
# Recover each token's owning request from its row-start
# tag. Ties (from zero-length requests) are harmless: those
# tokens have local length 0 and never dereference
# page_table.
req_idx = (
torch.searchsorted(
chunk.local_cu_seq_lens, chunk.cu_seqlen_ks, right=True
)
- 1
)
page_table = chunk.block_table[req_idx]
# The true max over this chunk's own rows, NOT
# chunk.max_local_total_seq_lens (that field sums every
# request's length in the chunk, bounding the old
# flat-gather buffer this paged path no longer allocates).
max_seq_len = int(local_seq_lens.max().item())
logits = ops.fp8_paged_mqa_logits_cpu(
q_slice,
kv_view,
weights[chunk.token_start : chunk.token_end],
local_seq_lens,
page_table,
block_size,
max_seq_len,
)
out_page_scratch = torch.empty(
(q_slice.shape[0], topk_tokens),
dtype=torch.int32,
device=kv_cache.device,
)
ops.topk_transform_512_cpu(
logits,
local_seq_lens,
page_table,
out_page_scratch,
block_size,
topk_indices,
)
if has_decode:
decode_metadata = attn_metadata_narrowed.decode
assert decode_metadata is not None
assert not decode_metadata.requires_padding, (
"CPU sparse indexer decode path does not support speculative "
"decoding (native MTP) yet."
)
batch_size = decode_metadata.decode_lens.shape[0]
if batch_size > 0:
# No native MTP on CPU (asserted above) => exactly one
# query token per decode request, so the flat slice below
# is already the batch-major layout
# fp8_paged_mqa_logits_cpu wants.
assert num_decode_tokens == batch_size, (
"CPU sparse indexer decode path expects exactly one query "
"token per decode request."
)
assert topk_tokens == 512, (
"topk_transform_512_cpu only supports index_topk == 512."
)
seq_lens = decode_metadata.seq_lens
seq_lens = (
seq_lens[:, -1].contiguous() if seq_lens.ndim == 2 else seq_lens
)
block_table = decode_metadata.block_table[:batch_size]
block_size = kv_cache.shape[1]
kv_view = kv_cache.view(kv_cache.shape[0], -1)
logits = ops.fp8_paged_mqa_logits_cpu(
q_quant[:num_decode_tokens],
kv_view,
weights[:num_decode_tokens],
seq_lens,
block_table,
block_size,
attn_metadata_narrowed.max_seq_len,
)
# out_page_indices is a required kernel output but unused:
# the indexer's topk output must stay local/compressed-
# context positions (resolved later by
# DeepseekV4CPUAttention.forward_mqa via
# map_local_to_global_slots_cpu).
out_page_scratch = torch.empty(
(batch_size, topk_tokens),
dtype=torch.int32,
device=kv_cache.device,
)
ops.topk_transform_512_cpu(
logits,
seq_lens,
block_table,
out_page_scratch,
block_size,
topk_indices_buffer[:num_decode_tokens, :topk_tokens],
)
return topk_indices_buffer