[Model] Add DeepSeek-V4 CPU backend (#55355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
This commit is contained in:
@@ -6,6 +6,9 @@ import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils import replace_parameter
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_upcast_e8m0_to_fp32,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
convert_to_channelwise,
|
||||
)
|
||||
@@ -281,10 +284,12 @@ class CPUFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
else params.weight_scale
|
||||
)
|
||||
assert weight_scale is not None
|
||||
if weight_scale.dtype in (torch.float8_e8m0fnu, torch.uint8):
|
||||
weight_scale = _upcast_e8m0_to_fp32(weight_scale.data)
|
||||
replace_parameter(
|
||||
layer,
|
||||
scale_attr,
|
||||
torch.nn.Parameter(weight_scale.data, requires_grad=False),
|
||||
torch.nn.Parameter(weight_scale.data.contiguous(), requires_grad=False),
|
||||
)
|
||||
|
||||
def apply_weights(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from .aiter import *
|
||||
from .cpu import *
|
||||
from .tilelang import *
|
||||
from .torch import *
|
||||
from .triton import *
|
||||
@@ -14,6 +15,9 @@ __all__ = [
|
||||
"mhc_post_aiter",
|
||||
"mhc_fused_post_pre_aiter",
|
||||
"hc_head_fused_aiter",
|
||||
"mhc_pre_cpu",
|
||||
"mhc_post_cpu",
|
||||
"hc_head_fused_cpu",
|
||||
"mhc_pre_tilelang",
|
||||
"mhc_post_tilelang",
|
||||
"mhc_fused_post_pre_tilelang",
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
|
||||
def mhc_pre_cpu(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""CPU-ported mHC pre block (see `mhc_pre_torch` for the eager reference).
|
||||
|
||||
The ported kernel (`hc_pre_fused_cpu`) only exposes one merged `hc_eps`
|
||||
(used for both `hc_pre_eps`/`hc_sinkhorn_eps`) and hardcodes the post-mix
|
||||
multiplier to 2.0 -- true of every real call site in
|
||||
`models/deepseek_v4/cpu/model.py`, so this is not a capability loss here.
|
||||
"""
|
||||
assert n_splits == 1, "mhc_pre_cpu does not support n_splits != 1"
|
||||
assert hc_pre_eps == hc_sinkhorn_eps, (
|
||||
"mhc_pre_cpu requires hc_pre_eps == hc_sinkhorn_eps (single merged hc_eps)"
|
||||
)
|
||||
assert hc_post_mult_value == 2.0, "mhc_pre_cpu hardcodes the post-mix multiplier"
|
||||
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
x_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
layer_input, post, comb = ops.hc_pre_fused_cpu(
|
||||
x_flat,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
hc_mult,
|
||||
sinkhorn_repeat,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
)
|
||||
|
||||
post_mix = post.view(*outer_shape, hc_mult, 1)
|
||||
comb_mix = comb.view(*outer_shape, hc_mult, hc_mult)
|
||||
layer_input = layer_input.view(*outer_shape, hidden_size)
|
||||
return post_mix, comb_mix, layer_input
|
||||
|
||||
|
||||
def mhc_post_cpu(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported mHC post block (see `mhc_post_torch` for the eager reference)."""
|
||||
hc_mult, hidden_size = residual.shape[-2:]
|
||||
outer_shape = residual.shape[:-2]
|
||||
|
||||
x_flat = x.reshape(-1, hidden_size)
|
||||
residual_flat = residual.reshape(-1, hc_mult, hidden_size)
|
||||
post_flat = post_layer_mix.reshape(-1, hc_mult).float()
|
||||
comb_flat = comb_res_mix.reshape(-1, hc_mult, hc_mult).float()
|
||||
|
||||
out = ops.hc_post_fused_cpu(x_flat, residual_flat, post_flat, comb_flat)
|
||||
return out.view(*outer_shape, hc_mult, hidden_size)
|
||||
|
||||
|
||||
def hc_head_fused_cpu(
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
"""CPU-ported HC head reduction (see `test_hc_head_cpu` in
|
||||
tests/kernels/test_mhc_kernels.py for the eager reference this is tested
|
||||
against).
|
||||
|
||||
The ported kernel's C++ signature takes `(hc_eps, norm_eps)` -- the
|
||||
opposite order from this wrapper's `(rms_norm_eps, hc_eps)`, which matches
|
||||
the real call site in `models/deepseek_v4/cpu/model.py`.
|
||||
"""
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
|
||||
|
||||
out = ops.hc_head_fused_cpu(hs_flat, hc_fn, hc_scale, hc_base, hc_eps, rms_norm_eps)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
@@ -218,7 +218,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import (
|
||||
select_experts,
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
|
||||
import math
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
@@ -34,6 +33,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
@@ -48,133 +50,26 @@ from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
# ===========================================================================
|
||||
# Routing
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def grouped_topk(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
num_expert_group: int = 0,
|
||||
topk_group: int = 0,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch"
|
||||
|
||||
gating_output = gating_output.float()
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(gating_output, dim=-1)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = gating_output.sigmoid()
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
num_token = scores.shape[0]
|
||||
if e_score_correction_bias is not None:
|
||||
original_scores = scores
|
||||
scores = scores + e_score_correction_bias.unsqueeze(0)
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
)
|
||||
else:
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
|
||||
) # [n, n_group]
|
||||
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
|
||||
1
|
||||
] # [n, top_k_group]
|
||||
group_mask = torch.zeros_like(group_scores) # [n, n_group]
|
||||
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
|
||||
.reshape(num_token, -1)
|
||||
) # [n, e]
|
||||
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
|
||||
topk_weights = original_scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights, topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def select_experts(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
use_grouped_topk: bool,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if use_grouped_topk:
|
||||
assert topk_group is not None
|
||||
assert num_expert_group is not None
|
||||
return grouped_topk(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
elif custom_routing_function is None:
|
||||
assert scoring_func == "softmax"
|
||||
topk_logit_vals, topk_idx = torch.topk(
|
||||
router_logits, k=top_k, dim=-1, sorted=False
|
||||
)
|
||||
if renormalize:
|
||||
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
|
||||
else:
|
||||
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
|
||||
topk_vals = (topk_logit_vals - logZ).exp()
|
||||
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
|
||||
else:
|
||||
topk_weights, topk_ids = custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=top_k,
|
||||
renormalize=renormalize,
|
||||
)
|
||||
# cpu_fused_moe reads routing tensors as contiguous float32/int32
|
||||
# buffers and does not account for tensor strides.
|
||||
topk_weights = topk_weights.to(torch.float32).contiguous()
|
||||
topk_ids = topk_ids.to(torch.int32).contiguous()
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Unquantized (BF16/FP16/FP32) MoE
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
class CPUUnquantizedExperts(mk.FusedMoEExpertsModular):
|
||||
"""Portable vector grouped-gemm unquantized MoE experts."""
|
||||
|
||||
isa = "vec"
|
||||
output_alignment = 32
|
||||
reduction_alignment = 1
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
self._use_zentorch = False
|
||||
|
||||
@classmethod
|
||||
def _intermediate_alignment(cls) -> int:
|
||||
return math.lcm(cls.output_alignment, cls.reduction_alignment)
|
||||
@@ -210,20 +105,6 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
)
|
||||
return True, None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
# Router configuration that the monolithic apply() signature cannot
|
||||
# carry. Captured off the layer in process_weights_after_loading.
|
||||
self.use_grouped_topk = False
|
||||
self.renormalize = False
|
||||
self.scoring_func = "softmax"
|
||||
self.custom_routing_function: Callable | None = None
|
||||
self._use_zentorch = False
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
@@ -286,8 +167,8 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
# Routing runs in select_experts(), which covers every routing method
|
||||
# a layer can be configured with, including custom routing functions.
|
||||
# Routing runs in the CPURouter, which covers every routing method a
|
||||
# layer can be configured with, including custom routing functions.
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
@@ -298,10 +179,6 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
return True
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.use_grouped_topk = layer.use_grouped_topk
|
||||
self.renormalize = layer.renormalize
|
||||
self.scoring_func = layer.scoring_func
|
||||
self.custom_routing_function = layer.custom_routing_function
|
||||
self._use_zentorch = is_zentorch_moe_supported(layer)
|
||||
if self._use_zentorch:
|
||||
return
|
||||
@@ -345,65 +222,44 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
# this very object, which is what feeds self.w1_bias in apply().
|
||||
layer.w13_bias.data = new_bias
|
||||
|
||||
def _select_experts(
|
||||
def workspace_shapes(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
num_expert_group: int | None,
|
||||
topk_group: int | None,
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
use_grouped_topk=self.use_grouped_topk,
|
||||
renormalize=self.renormalize,
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
custom_routing_function=self.custom_routing_function,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# cpu_fused_moe manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = self._select_experts(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
num_expert_group,
|
||||
topk_group,
|
||||
e_score_correction_bias,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
) -> None:
|
||||
# apply_router_weight_on_input (topk=1 only) is already applied to
|
||||
# hidden_states by MoEPrepareAndFinalizeNoDPEPModular.prepare().
|
||||
if self._use_zentorch:
|
||||
output = torch.empty_like(hidden_states)
|
||||
torch.ops.zentorch.zentorch_fused_moe(
|
||||
output,
|
||||
hidden_states,
|
||||
@@ -416,9 +272,9 @@ class CPUUnquantizedExperts(mk.FusedMoEExpertsMonolithic):
|
||||
apply_router_weight_on_input,
|
||||
str(activation.value).lower(),
|
||||
)
|
||||
return output
|
||||
|
||||
return cpu_fused_moe(
|
||||
return
|
||||
cpu_fused_moe(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
@@ -460,8 +316,6 @@ class X86CPUUnquantizedExperts(CPUUnquantizedExperts):
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if is_zentorch_moe_config_supported(moe_config):
|
||||
return True, None
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
return False, "kernel requires bfloat16 activations"
|
||||
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
|
||||
@@ -496,8 +350,6 @@ class ArmCPUUnquantizedExperts(CPUUnquantizedExperts):
|
||||
)
|
||||
if not supported:
|
||||
return supported, reason
|
||||
if is_zentorch_moe_config_supported(moe_config):
|
||||
return True, None
|
||||
if moe_config.in_dtype != torch.bfloat16:
|
||||
return False, "kernel requires bfloat16 activations"
|
||||
cpu_cls = cast(type[CPUUnquantizedExperts], cls)
|
||||
@@ -552,8 +404,8 @@ def prepare_fp8_moe_layer_for_cpu(
|
||||
return packed_w13, packed_w2
|
||||
|
||||
|
||||
class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU FP8 W8A16 block-quantized monolithic MoE experts."""
|
||||
class CPUExpertsFp8(mk.FusedMoEExpertsModular):
|
||||
"""CPU FP8 W8A16 block-quantized modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -615,6 +467,7 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -624,42 +477,49 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# fused_experts_cpu manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
# fused_experts_cpu always applies topk_weights internally on
|
||||
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
|
||||
# also pre-apply it to hidden_states, double-weighting the
|
||||
# output. Not needed by any CPU FP8 model today.
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsFp8 does not support apply_router_weight_on_input=True."
|
||||
)
|
||||
block_shape = (
|
||||
list(self.quant_config.block_shape)
|
||||
if self.quant_config.block_shape
|
||||
@@ -670,13 +530,13 @@ class CPUExpertsFp8(mk.FusedMoEExpertsMonolithic):
|
||||
)
|
||||
)
|
||||
|
||||
return fused_experts_cpu(
|
||||
fused_experts_cpu(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.FP8_W8A16, # moe_comp_method
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
@@ -710,8 +570,8 @@ def prepare_mxfp4_moe_layer_for_cpu(
|
||||
return packed_w13, packed_w2, packed_w13_scale, packed_w2_scale
|
||||
|
||||
|
||||
class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU MXFP4 W4A16 monolithic MoE experts."""
|
||||
class CPUExpertsMxfp4(mk.FusedMoEExpertsModular):
|
||||
"""CPU MXFP4 W4A16 modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -773,6 +633,7 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
RoutingMethodType.Default,
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
RoutingMethodType.DeepseekV4,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
@@ -782,55 +643,62 @@ class CPUExpertsMxfp4(mk.FusedMoEExpertsMonolithic):
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# fused_experts_cpu manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
# fused_experts_cpu always applies topk_weights internally on
|
||||
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
|
||||
# also pre-apply it to hidden_states, double-weighting the
|
||||
# output. Not needed by any CPU MXFP4 model today.
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsMxfp4 does not support apply_router_weight_on_input=True."
|
||||
)
|
||||
# Get bias and swiglu params from quant config
|
||||
w1_bias = self.quant_config.w1_bias
|
||||
w2_bias = self.quant_config.w2_bias
|
||||
alpha = getattr(self.quant_config, "gemm1_alpha", None)
|
||||
limit = getattr(self.quant_config, "gemm1_clamp_limit", None)
|
||||
|
||||
return fused_experts_cpu(
|
||||
fused_experts_cpu(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.MXFP4, # moe_comp_method
|
||||
self.w1_scale, # w1_scale
|
||||
self.w2_scale, # w2_scale
|
||||
@@ -911,8 +779,8 @@ def prepare_int4_moe_layer_for_cpu(
|
||||
return (blocked_w13, blocked_w2, blocked_s13, blocked_s2, blocked_z13, blocked_z2)
|
||||
|
||||
|
||||
class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
"""CPU INT4 W4A16 group-quantized monolithic MoE experts.
|
||||
class CPUExpertsInt4(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT4 W4A16 group-quantized modular MoE experts.
|
||||
|
||||
Weights are int4 (packed), activations are bf16/fp16.
|
||||
Internally uses int8 compute via fused_experts_cpu with INT4_W4A8.
|
||||
@@ -992,55 +860,75 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
# convert_weight_packed_scale_zp blocks w1/w2 into AMX tiles
|
||||
# (E, Nc, Kc, buffer_bytes), not the (E, N, K) layout the base
|
||||
# implementation assumes -- N/K aren't recoverable from that shape,
|
||||
# so read them from moe_config instead.
|
||||
E = w1.shape[0]
|
||||
K = a1.size(-1)
|
||||
N = (
|
||||
self.moe_config.intermediate_size_per_partition
|
||||
* self.moe_config.w13_num_shards
|
||||
)
|
||||
M = a1.size(0) if a1.dim() == 2 else a1.size(1)
|
||||
topk = topk_ids.size(1)
|
||||
return E, M, N, K, topk
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# fused_experts_cpu manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsInt4 (W4A16) does not support "
|
||||
"apply_router_weight_on_input=True. "
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
return fused_experts_cpu(
|
||||
fused_experts_cpu(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.INT4_W4A8,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
@@ -1060,9 +948,9 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
class CPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""CPU INT8 W8A8 per-channel weight / dynamic per-token activation
|
||||
monolithic MoE experts."""
|
||||
modular MoE experts."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1172,49 +1060,56 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# fused_experts_cpu manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
# grouped topk + fused topk bias parameters
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
return fused_experts_cpu(
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
# fused_experts_cpu always applies topk_weights internally on
|
||||
# combine; MoEPrepareAndFinalizeNoDPEPModular.prepare() would
|
||||
# also pre-apply it to hidden_states, double-weighting the
|
||||
# output. Not needed by any CPU INT8 W8A8 model today.
|
||||
raise NotImplementedError(
|
||||
"CPUExpertsInt8 does not support apply_router_weight_on_input=True."
|
||||
)
|
||||
fused_experts_cpu(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
False, # inplace
|
||||
CPUQuantMethod.INT8_W8A8,
|
||||
self.w1_scale,
|
||||
self.w2_scale,
|
||||
@@ -1229,7 +1124,7 @@ class CPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
)
|
||||
|
||||
|
||||
class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
class ArmCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""Arm INT8 MoE with per-token activation and channelwise weight quantization."""
|
||||
|
||||
@property
|
||||
@@ -1331,48 +1226,47 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# cpu_fused_moe_int8 manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1
|
||||
hidden_states.mul_(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
) -> None:
|
||||
# apply_router_weight_on_input (topk=1 only) is already applied to
|
||||
# hidden_states by MoEPrepareAndFinalizeNoDPEPModular.prepare().
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
return cpu_fused_moe_int8(
|
||||
cpu_fused_moe_int8(
|
||||
output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
@@ -1388,7 +1282,7 @@ class ArmCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
)
|
||||
|
||||
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"""AMD Zen INT8 MoE with per-token activation and channelwise weight
|
||||
quantization, dispatched through zentorch."""
|
||||
|
||||
@@ -1424,9 +1318,7 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
if self.w2_bias is None
|
||||
else self.w2_bias.detach().to(torch.bfloat16).contiguous()
|
||||
)
|
||||
logger.info_once(
|
||||
"[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE (monolithic experts)"
|
||||
)
|
||||
logger.info_once("[zen_cpu] Using zentorch_fused_moe for W8A8 INT8 MoE")
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
@@ -1487,48 +1379,47 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
def supports_expert_map(self) -> bool:
|
||||
return False
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# zentorch_fused_moe manages its own scratch space.
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
num_expert_group: int | None = None,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
topk_weights, topk_ids = select_experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
use_grouped_topk=num_expert_group is not None,
|
||||
top_k=self.moe_config.experts_per_token,
|
||||
renormalize=self.moe_config.routing_method
|
||||
in (
|
||||
RoutingMethodType.Renormalize,
|
||||
RoutingMethodType.RenormalizeNaive,
|
||||
),
|
||||
topk_group=topk_group,
|
||||
num_expert_group=num_expert_group,
|
||||
scoring_func="softmax",
|
||||
routed_scaling_factor=(
|
||||
routed_scaling_factor if routed_scaling_factor is not None else 1.0
|
||||
),
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
) -> None:
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input is only implemented for topk=1"
|
||||
)
|
||||
hidden_states = hidden_states.mul(topk_weights.to(hidden_states.dtype))
|
||||
|
||||
output = torch.empty_like(hidden_states)
|
||||
torch.ops.zentorch.zentorch_fused_moe(
|
||||
output,
|
||||
hidden_states,
|
||||
@@ -1543,4 +1434,3 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsMonolithic):
|
||||
self._w1_scale_bf16,
|
||||
self._w2_scale_bf16,
|
||||
)
|
||||
return output
|
||||
|
||||
@@ -1175,6 +1175,24 @@ class FusedMoEKernelModularImpl:
|
||||
# time we need cache3, we're done with cache1.
|
||||
# Reuse workspace13 for the output since there is only one chunk.
|
||||
max_shape_size = max(prod(workspace13_shape), prod(fused_out_shape))
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# Every CPU FusedMoEExpertsModular kernel reports zero-sized
|
||||
# workspace13/workspace2 (it manages its own scratch space
|
||||
# internally) and just needs the output buffer, so skip the
|
||||
# workspace manager entirely here -- it's the one part of this
|
||||
# call graph Dynamo can't trace (ContextVar-based lane lookup),
|
||||
# and CPU never actually needs its cross-chunk memory reuse.
|
||||
common_workspace = torch.empty(
|
||||
(max_shape_size,), dtype=workspace_dtype, device=device
|
||||
)
|
||||
workspace2 = torch.empty(
|
||||
workspace2_shape, dtype=workspace_dtype, device=device
|
||||
)
|
||||
workspace13 = _resize_cache(common_workspace, workspace13_shape)
|
||||
fused_out = _resize_cache(common_workspace, fused_out_shape)
|
||||
return workspace13, workspace2, fused_out
|
||||
|
||||
common_workspace, workspace2 = current_workspace_manager().get_simultaneous(
|
||||
((max_shape_size,), workspace_dtype),
|
||||
(workspace2_shape, workspace_dtype),
|
||||
|
||||
@@ -348,6 +348,7 @@ def _get_priority_backends_for_gpt_oss() -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
Mxfp4MoeBackend.BATCHED_MARLIN,
|
||||
Mxfp4MoeBackend.XPU,
|
||||
Mxfp4MoeBackend.CPU,
|
||||
Mxfp4MoeBackend.EMULATION,
|
||||
]
|
||||
return _AVAILABLE_BACKENDS
|
||||
@@ -366,6 +367,8 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]:
|
||||
]
|
||||
if current_platform.is_xpu():
|
||||
return [Mxfp4MoeBackend.XPU]
|
||||
if current_platform.is_cpu():
|
||||
return [Mxfp4MoeBackend.CPU]
|
||||
_AVAILABLE_BACKENDS = [
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
|
||||
Mxfp4MoeBackend.DEEPGEMM_MXFP4,
|
||||
@@ -627,17 +630,6 @@ def select_mxfp4_moe_backend(
|
||||
activation_format,
|
||||
)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
backend = Mxfp4MoeBackend.CPU
|
||||
logger.info_once(_make_log_backend(backend))
|
||||
return _return_or_raise(
|
||||
Mxfp4MoeBackend.CPU,
|
||||
config,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
unsupported_log = "; ".join(
|
||||
[
|
||||
f"backend: {backend.value}, reason: {reason}"
|
||||
@@ -1813,11 +1805,36 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=_cache_permute_indices,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_mxfp4_moe_layer_for_cpu,
|
||||
)
|
||||
|
||||
packed_w13, packed_w2, packed_w13_scale, packed_w2_scale = (
|
||||
prepare_mxfp4_moe_layer_for_cpu(
|
||||
w13_weight.data,
|
||||
w2_weight.data,
|
||||
w13_weight_scale.data,
|
||||
w2_weight_scale.data,
|
||||
)
|
||||
)
|
||||
if w13_bias is not None:
|
||||
w13_bias = w13_bias.data.to(torch.float32)
|
||||
if w2_bias is not None:
|
||||
w2_bias = w2_bias.data.to(torch.float32)
|
||||
return (
|
||||
packed_w13,
|
||||
packed_w2,
|
||||
packed_w13_scale,
|
||||
packed_w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported mxfp4_backend for Mxfp4MoEMethod: {mxfp4_backend}. "
|
||||
"Expected TRTLLM, FlashInfer CUTLASS, Triton, AITER, XPU, or "
|
||||
"emulation backend."
|
||||
"Expected TRTLLM, FlashInfer CUTLASS, Triton, AITER, XPU, "
|
||||
"CPU, or emulation backend."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Router for CPU MoE experts.
|
||||
|
||||
This is the routing logic that used to live inline in each CPU
|
||||
FusedMoEExpertsMonolithic.apply() (see git history of
|
||||
experts/cpu_moe.py); factoring it out into a proper FusedMoERouter is what
|
||||
lets CPU MoE experts be FusedMoEExpertsModular like every other backend.
|
||||
The math below is unchanged from that prior inline version -- this is a
|
||||
relocation, not a rewrite.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import biased_topk_cpu, hash_topk_cpu
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
|
||||
|
||||
|
||||
def _grouped_topk(
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
num_expert_group: int,
|
||||
topk_group: int,
|
||||
scoring_func: str,
|
||||
routed_scaling_factor: float,
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
gating_output = gating_output.float()
|
||||
if scoring_func == "softmax":
|
||||
scores = torch.softmax(gating_output, dim=-1)
|
||||
elif scoring_func == "sigmoid":
|
||||
scores = gating_output.sigmoid()
|
||||
else:
|
||||
raise ValueError(f"Unsupported scoring function: {scoring_func}")
|
||||
|
||||
num_token = scores.shape[0]
|
||||
if e_score_correction_bias is not None:
|
||||
original_scores = scores
|
||||
scores = scores + e_score_correction_bias.unsqueeze(0)
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).topk(2, dim=-1)[0].sum(dim=-1)
|
||||
)
|
||||
else:
|
||||
group_scores = (
|
||||
scores.view(num_token, num_expert_group, -1).max(dim=-1).values
|
||||
) # [n, n_group]
|
||||
group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
|
||||
1
|
||||
] # [n, top_k_group]
|
||||
group_mask = torch.zeros_like(group_scores) # [n, n_group]
|
||||
group_mask.scatter_(1, group_idx, 1) # [n, n_group]
|
||||
score_mask = (
|
||||
group_mask.unsqueeze(-1)
|
||||
.expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
|
||||
.reshape(num_token, -1)
|
||||
) # [n, e]
|
||||
tmp_scores = scores.masked_fill(~score_mask.bool(), float("-inf")) # [n, e]
|
||||
|
||||
if e_score_correction_bias is not None:
|
||||
topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)[1]
|
||||
topk_weights = original_scores.gather(1, topk_ids)
|
||||
else:
|
||||
topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
|
||||
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights, topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def _sqrtsoftplus_bias_topk(
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
renormalize: bool,
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float,
|
||||
input_ids: torch.Tensor | None,
|
||||
hash_indices_table: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""DeepSeek V4's routing: weight = sqrt(softplus(logit)); expert
|
||||
selection uses a bias-corrected score (weight + correction_bias), but
|
||||
the returned weight for each selected expert is the *unbiased* value
|
||||
(the "noaux_tc" pattern -- bias steers selection only). Always
|
||||
renormalizes selected weights to sum to ``routed_scaling_factor``,
|
||||
matching the reference Triton kernel
|
||||
(vllm/model_executor/layers/fused_moe/router/dsv4_topk.py).
|
||||
|
||||
When ``hash_indices_table`` is given (hash-routed layers), expert
|
||||
selection instead comes from ``hash_indices_table[input_ids]`` --
|
||||
the bias-corrected-score topk is skipped entirely, matching the CUDA
|
||||
reference kernel (``dsv4HashTopkSoftplusSqrt``).
|
||||
|
||||
Backed by the ported SGLang AVX512 kernels `biased_topk_cpu` (the
|
||||
correction-bias path) and `hash_topk_cpu` (the hash-routed-layer path);
|
||||
see csrc/cpu/sgl-kernels/topk.cpp.
|
||||
"""
|
||||
assert renormalize, "DeepseekV4 routing always renormalizes"
|
||||
gating_output = router_logits.contiguous()
|
||||
num_experts = gating_output.shape[1]
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None
|
||||
# hash_indices_table is int32 by construction (see cpu/model.py) and
|
||||
# advanced indexing always allocates a fresh contiguous tensor.
|
||||
tid2eid = hash_indices_table[input_ids]
|
||||
return hash_topk_cpu(
|
||||
gating_output,
|
||||
tid2eid,
|
||||
top_k,
|
||||
"sqrtsoftplus",
|
||||
0, # num_fused_shared_experts: never fused into routing on CPU
|
||||
num_experts,
|
||||
routed_scaling_factor,
|
||||
)
|
||||
assert e_score_correction_bias is not None
|
||||
return biased_topk_cpu(
|
||||
gating_output, # only used for num_tokens/device (dtype forced fp32)
|
||||
gating_output,
|
||||
e_score_correction_bias.contiguous(),
|
||||
top_k,
|
||||
renormalize,
|
||||
"sqrtsoftplus",
|
||||
0, # num_fused_shared_experts: never fused into routing on CPU
|
||||
routed_scaling_factor,
|
||||
True, # apply_routed_scaling_factor_on_output
|
||||
)
|
||||
|
||||
|
||||
def _softmax_topk(
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
renormalize: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
topk_logit_vals, topk_idx = torch.topk(router_logits, k=top_k, dim=-1, sorted=False)
|
||||
if renormalize:
|
||||
topk_vals = torch.softmax(topk_logit_vals, dim=-1)
|
||||
else:
|
||||
logZ = torch.logsumexp(router_logits, dim=-1, keepdim=True)
|
||||
topk_vals = (topk_logit_vals - logZ).exp()
|
||||
return topk_vals.to(torch.float32), topk_idx.to(torch.int32)
|
||||
|
||||
|
||||
class CPURouter(BaseRouter):
|
||||
"""Router covering every routing scheme CPU MoE experts support: plain
|
||||
softmax top-k, grouped top-k (with an optional correction bias), an
|
||||
arbitrary custom_routing_function, and DeepSeek V4's sqrtsoftplus scheme
|
||||
(bias-corrected or hash-routed via the ported biased_topk_cpu/
|
||||
hash_topk_cpu kernels). All of the above except sqrtsoftplus are plain
|
||||
torch ops so Dynamo traces straight through into them like it did when
|
||||
this logic lived inline in FusedMoEExpertsMonolithic.apply() -- no new
|
||||
compile boundary is introduced here.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
top_k: int,
|
||||
global_num_experts: int,
|
||||
use_grouped_topk: bool = False,
|
||||
num_expert_group: int | None = None,
|
||||
topk_group: int | None = None,
|
||||
renormalize: bool = True,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
eplb_state: EplbLayerState | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
self.use_grouped_topk = use_grouped_topk
|
||||
self.num_expert_group = num_expert_group
|
||||
self.topk_group = topk_group
|
||||
self.renormalize = renormalize
|
||||
self.scoring_func = scoring_func
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
self.custom_routing_function = custom_routing_function
|
||||
self._hash_indices_table = hash_indices_table
|
||||
|
||||
@property
|
||||
def routing_method_type(self) -> RoutingMethodType:
|
||||
return get_routing_method_type(
|
||||
scoring_func=self.scoring_func,
|
||||
top_k=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
has_e_score_bias=self.e_score_correction_bias is not None,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
)
|
||||
|
||||
def _compute_routing(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
indices_type: torch.dtype | None,
|
||||
*,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.use_grouped_topk:
|
||||
assert self.topk_group is not None
|
||||
assert self.num_expert_group is not None
|
||||
return _grouped_topk(
|
||||
gating_output=router_logits,
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
num_expert_group=self.num_expert_group,
|
||||
topk_group=self.topk_group,
|
||||
scoring_func=self.scoring_func,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=self.e_score_correction_bias,
|
||||
)
|
||||
elif self.scoring_func == "sqrtsoftplus":
|
||||
assert (
|
||||
self._hash_indices_table is not None
|
||||
or self.e_score_correction_bias is not None
|
||||
)
|
||||
return _sqrtsoftplus_bias_topk(
|
||||
router_logits=router_logits,
|
||||
top_k=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
e_score_correction_bias=self.e_score_correction_bias,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=self._hash_indices_table,
|
||||
)
|
||||
elif self.custom_routing_function is None:
|
||||
assert self.scoring_func == "softmax"
|
||||
return _softmax_topk(router_logits, self.top_k, self.renormalize)
|
||||
else:
|
||||
topk_weights, topk_ids = self.custom_routing_function(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
topk=self.top_k,
|
||||
renormalize=self.renormalize,
|
||||
)
|
||||
# cpu_fused_moe reads routing tensors as contiguous float32/int32
|
||||
# buffers and does not account for tensor strides.
|
||||
topk_weights = topk_weights.to(torch.float32).contiguous()
|
||||
topk_ids = topk_ids.to(torch.int32).contiguous()
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def select_experts(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
top_k: int,
|
||||
use_grouped_topk: bool,
|
||||
renormalize: bool,
|
||||
topk_group: int | None = None,
|
||||
num_expert_group: int | None = None,
|
||||
custom_routing_function: Callable | None = None,
|
||||
scoring_func: str = "softmax",
|
||||
routed_scaling_factor: float = 1.0,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Routing helper for the CPU MoE experts that still use the monolithic
|
||||
contract and so must compute routing themselves from ``router_logits``
|
||||
(the zentorch-accelerated path and the ARM W4A8 dynamic-quant kernel,
|
||||
kept monolithic since zentorch/Arm hardware isn't available to validate
|
||||
a modular migration here). Modular CPU experts get routing from
|
||||
``CPURouter`` instead; this is a thin wrapper around the same logic for
|
||||
the two classes that can't use it.
|
||||
"""
|
||||
router = CPURouter(
|
||||
top_k=top_k,
|
||||
global_num_experts=router_logits.shape[-1],
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
custom_routing_function=custom_routing_function,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
return router._compute_routing(
|
||||
hidden_states, router_logits, None, input_ids=input_ids
|
||||
)
|
||||
@@ -14,6 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.router.aiter_shared_routed_fused_moe_router import ( # noqa: E501
|
||||
AiterSharedRoutedFusedMoERouter,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.cpu_router import CPURouter
|
||||
from vllm.model_executor.layers.fused_moe.router.custom_routing_router import (
|
||||
CustomRoutingRouter,
|
||||
)
|
||||
@@ -35,6 +36,7 @@ from vllm.model_executor.layers.fused_moe.router.routing_simulator_router import
|
||||
from vllm.model_executor.layers.fused_moe.router.zero_expert_router import (
|
||||
ZeroExpertRouter,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def create_fused_moe_router(
|
||||
@@ -141,6 +143,26 @@ def create_fused_moe_router(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# CPURouter covers every routing scheme CPU MoE experts support
|
||||
# (plain softmax, grouped topk, custom routing functions, and
|
||||
# DeepSeek V4's sqrtsoftplus/hash routing) in a single class, so it
|
||||
# takes priority over the scheme-specific routers below.
|
||||
return CPURouter(
|
||||
top_k=top_k,
|
||||
global_num_experts=global_num_experts,
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
renormalize=renormalize,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
custom_routing_function=custom_routing_function,
|
||||
hash_indices_table=hash_indices_table,
|
||||
eplb_state=eplb_state,
|
||||
)
|
||||
|
||||
if use_grouped_topk:
|
||||
assert custom_routing_function is None
|
||||
if num_expert_group is None or topk_group is None:
|
||||
|
||||
@@ -317,10 +317,17 @@ class MoERunner(MoERunnerInterface):
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu() or current_platform.is_cpu():
|
||||
if current_platform.is_tpu():
|
||||
# TODO: Once the OOM issue for the TPU backend is resolved, we
|
||||
# will switch to using the moe_forward custom op.
|
||||
# Note: CPU doesn't require wrapped _forward_impl.
|
||||
return _moe_forward if self._shared_experts is None else _moe_forward_shared
|
||||
|
||||
if current_platform.is_cpu():
|
||||
# CPU never touches the workspace manager (Monolithic experts
|
||||
# skip it entirely; Modular experts' _allocate_buffers bypasses
|
||||
# it too, see modular_kernel.py) -- the ContextVar-based lane
|
||||
# lookup was the only part of this call graph Dynamo can't
|
||||
# trace, so CPU can always call the fused-MoE op directly.
|
||||
return _moe_forward if self._shared_experts is None else _moe_forward_shared
|
||||
|
||||
return (
|
||||
|
||||
@@ -261,6 +261,36 @@ class MHCPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return mhc_kernels.mhc_pre_cpu(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
n_splits,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_post]
|
||||
@CustomOp.register("mhc_post")
|
||||
@@ -337,6 +367,15 @@ class MHCPostOp(CustomOp):
|
||||
comb_res_mix,
|
||||
)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return mhc_kernels.mhc_post_cpu(x, residual, post_layer_mix, comb_res_mix)
|
||||
|
||||
|
||||
# --8<-- [start:hc_head]
|
||||
@CustomOp.register("hc_head")
|
||||
@@ -444,6 +483,19 @@ class HCHeadOp(CustomOp):
|
||||
)
|
||||
return out.view(*outer_shape, hidden_size)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
hc_fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
hc_eps: float,
|
||||
) -> torch.Tensor:
|
||||
return mhc_kernels.hc_head_fused_cpu(
|
||||
hidden_states, hc_fn, hc_scale, hc_base, rms_norm_eps, hc_eps
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mhc_fused_post_pre]
|
||||
@CustomOp.register("mhc_fused_post_pre")
|
||||
@@ -654,6 +706,42 @@ class MHCFusedPostPreOp(CustomOp):
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
n_splits: int = 1,
|
||||
tile_n: int = 1,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 0.0,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Decompose into post + pre (no fused kernel available).
|
||||
residual_cur = mhc_kernels.mhc_post_cpu(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
post_mix_cur, comb_mix_cur, layer_input_cur = mhc_kernels.mhc_pre_cpu(
|
||||
residual_cur,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
return residual_cur, post_mix_cur, comb_mix_cur, layer_input_cur
|
||||
|
||||
|
||||
def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor:
|
||||
"""[s, hidden_size] -> [s, n * hidden_size] by replication."""
|
||||
|
||||
@@ -828,10 +828,12 @@ class SparseAttnIndexer(CustomOp):
|
||||
return self.forward_cuda(hidden_states, q_quant, k, weights)
|
||||
elif current_platform.is_rocm():
|
||||
return self.forward_hip(hidden_states, q_quant, k, weights)
|
||||
elif current_platform.is_cpu():
|
||||
return self.forward_cpu(hidden_states, q_quant, k, weights)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexer native forward is only implemented for "
|
||||
"CUDA, ROCm and XPU platforms."
|
||||
"CUDA, ROCm, XPU and CPU platforms."
|
||||
)
|
||||
|
||||
def forward_cuda(
|
||||
@@ -919,3 +921,196 @@ class SparseAttnIndexer(CustomOp):
|
||||
"Sparse attention indexer ROCm path is only supported on AITER. "
|
||||
"Please enable aiter with VLLM_ROCM_USE_AITER=1"
|
||||
)
|
||||
|
||||
def forward_cpu(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
k: torch.Tensor | None,
|
||||
weights: torch.Tensor,
|
||||
):
|
||||
"""CPU sparse attention indexer: cache write stays eager Python glue
|
||||
(own K-cache layout, not shared with the main attention cache
|
||||
write). PREFILL and DECODE both call the ported
|
||||
``fp8_paged_mqa_logits_cpu``/``topk_transform_512_cpu`` kernels,
|
||||
which read the paged K-cache directly via ``page_table`` -- no
|
||||
eager gather step, no per-request Python loop.
|
||||
|
||||
``prefill_metadata.chunks`` always has exactly one entry here:
|
||||
``DeepseekV4CPUIndexerMetadataBuilder`` overrides the base chunk
|
||||
split to always return the whole step's prefill batch as one
|
||||
chunk, since the base chunking only bounds CUDA/XPU's dense M*N
|
||||
logits tensor and flat K-gather workspace, neither of which this
|
||||
paged kernel allocates.
|
||||
"""
|
||||
assert not self.use_fp4_cache, (
|
||||
"CPU sparse indexer doesn't support fp4 cache yet"
|
||||
)
|
||||
assert isinstance(q_quant, torch.Tensor), (
|
||||
"CPU sparse_attn_indexer expects a single FP8 q_quant tensor"
|
||||
)
|
||||
assert self.dcp_world_size <= 1 and not self.use_pcp, (
|
||||
"CPU sparse indexer doesn't support decode/prefill context parallelism yet."
|
||||
)
|
||||
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
attn_metadata_narrowed: DeepseekV32IndexerMetadata | None = None
|
||||
if isinstance(attn_metadata, dict):
|
||||
metadata = attn_metadata[self.k_cache.prefix]
|
||||
assert isinstance(metadata, DeepseekV32IndexerMetadata)
|
||||
attn_metadata_narrowed = metadata
|
||||
if attn_metadata_narrowed is None:
|
||||
# Profiling/dummy run: no real metadata to act on.
|
||||
return self.topk_indices_buffer
|
||||
|
||||
kv_cache = self.k_cache.kv_cache
|
||||
topk_tokens = self.topk_tokens
|
||||
topk_indices_buffer = self.topk_indices_buffer
|
||||
slot_mapping = attn_metadata_narrowed.slot_mapping
|
||||
has_decode = attn_metadata_narrowed.num_decodes > 0
|
||||
has_prefill = attn_metadata_narrowed.num_prefills > 0
|
||||
num_decode_tokens = attn_metadata_narrowed.num_decode_tokens
|
||||
|
||||
num_tokens = slot_mapping.shape[0]
|
||||
if k is not None:
|
||||
k = k[:num_tokens]
|
||||
|
||||
if not self.skip_k_cache_insert:
|
||||
# Only reachable via DeepseekV32Attention with
|
||||
# prefill_context_parallel_size > 1 on CPU -- set_k_cpu/set_s_cpu
|
||||
# (the kernels this used to call) have been removed as unused/
|
||||
# untested (csrc/cpu/sgl-kernels/store_cache.cpp).
|
||||
raise NotImplementedError(
|
||||
"SparseAttnIndexer.forward_cpu: skip_k_cache_insert=False "
|
||||
"(prefill context parallel on CPU) is not supported."
|
||||
)
|
||||
|
||||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||||
|
||||
if has_prefill:
|
||||
assert topk_tokens == 512, (
|
||||
"topk_transform_512_cpu only supports index_topk == 512."
|
||||
)
|
||||
prefill_metadata = attn_metadata_narrowed.prefill
|
||||
assert prefill_metadata is not None
|
||||
assert len(prefill_metadata.chunks) == 1, (
|
||||
"forward_cpu expects the prefill metadata builder to always "
|
||||
"produce a single chunk -- see "
|
||||
"DeepseekV4CPUIndexerMetadataBuilder._split_indexer_prefill_chunks."
|
||||
)
|
||||
chunk = prefill_metadata.chunks[0]
|
||||
# kv_cache is a per-layer view into vLLM's shared multi-layer
|
||||
# cache allocation, so its block stride generally exceeds
|
||||
# block_size * page_width; fp8_paged_mqa_logits_cpu reads
|
||||
# kv_view.stride(0) explicitly, so no copy is needed here.
|
||||
kv_view = kv_cache.view(kv_cache.shape[0], -1)
|
||||
block_size = kv_cache.shape[1]
|
||||
q_slice = q_quant[chunk.token_start : chunk.token_end]
|
||||
topk_indices = topk_indices_buffer[
|
||||
chunk.token_start : chunk.token_end, :topk_tokens
|
||||
]
|
||||
if chunk.local_total_seq_lens == 0:
|
||||
topk_indices.fill_(-1)
|
||||
else:
|
||||
assert chunk.local_cu_seq_lens is not None
|
||||
# Each token's own (per-request, DCP-local) causal length.
|
||||
local_seq_lens = chunk.cu_seqlen_ke - chunk.cu_seqlen_ks
|
||||
# Recover each token's owning request from its row-start
|
||||
# tag. Ties (from zero-length requests) are harmless: those
|
||||
# tokens have local length 0 and never dereference
|
||||
# page_table.
|
||||
req_idx = (
|
||||
torch.searchsorted(
|
||||
chunk.local_cu_seq_lens, chunk.cu_seqlen_ks, right=True
|
||||
)
|
||||
- 1
|
||||
)
|
||||
page_table = chunk.block_table[req_idx]
|
||||
# The true max over this chunk's own rows, NOT
|
||||
# chunk.max_local_total_seq_lens (that field sums every
|
||||
# request's length in the chunk, bounding the old
|
||||
# flat-gather buffer this paged path no longer allocates).
|
||||
max_seq_len = int(local_seq_lens.max().item())
|
||||
|
||||
logits = ops.fp8_paged_mqa_logits_cpu(
|
||||
q_slice,
|
||||
kv_view,
|
||||
weights[chunk.token_start : chunk.token_end],
|
||||
local_seq_lens,
|
||||
page_table,
|
||||
block_size,
|
||||
max_seq_len,
|
||||
)
|
||||
out_page_scratch = torch.empty(
|
||||
(q_slice.shape[0], topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
ops.topk_transform_512_cpu(
|
||||
logits,
|
||||
local_seq_lens,
|
||||
page_table,
|
||||
out_page_scratch,
|
||||
block_size,
|
||||
topk_indices,
|
||||
)
|
||||
|
||||
if has_decode:
|
||||
decode_metadata = attn_metadata_narrowed.decode
|
||||
assert decode_metadata is not None
|
||||
assert not decode_metadata.requires_padding, (
|
||||
"CPU sparse indexer decode path does not support speculative "
|
||||
"decoding (native MTP) yet."
|
||||
)
|
||||
batch_size = decode_metadata.decode_lens.shape[0]
|
||||
if batch_size > 0:
|
||||
# No native MTP on CPU (asserted above) => exactly one
|
||||
# query token per decode request, so the flat slice below
|
||||
# is already the batch-major layout
|
||||
# fp8_paged_mqa_logits_cpu wants.
|
||||
assert num_decode_tokens == batch_size, (
|
||||
"CPU sparse indexer decode path expects exactly one query "
|
||||
"token per decode request."
|
||||
)
|
||||
assert topk_tokens == 512, (
|
||||
"topk_transform_512_cpu only supports index_topk == 512."
|
||||
)
|
||||
seq_lens = decode_metadata.seq_lens
|
||||
seq_lens = (
|
||||
seq_lens[:, -1].contiguous() if seq_lens.ndim == 2 else seq_lens
|
||||
)
|
||||
block_table = decode_metadata.block_table[:batch_size]
|
||||
block_size = kv_cache.shape[1]
|
||||
kv_view = kv_cache.view(kv_cache.shape[0], -1)
|
||||
|
||||
logits = ops.fp8_paged_mqa_logits_cpu(
|
||||
q_quant[:num_decode_tokens],
|
||||
kv_view,
|
||||
weights[:num_decode_tokens],
|
||||
seq_lens,
|
||||
block_table,
|
||||
block_size,
|
||||
attn_metadata_narrowed.max_seq_len,
|
||||
)
|
||||
|
||||
# out_page_indices is a required kernel output but unused:
|
||||
# the indexer's topk output must stay local/compressed-
|
||||
# context positions (resolved later by
|
||||
# DeepseekV4CPUAttention.forward_mqa via
|
||||
# map_local_to_global_slots_cpu).
|
||||
out_page_scratch = torch.empty(
|
||||
(batch_size, topk_tokens),
|
||||
dtype=torch.int32,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
ops.topk_transform_512_cpu(
|
||||
logits,
|
||||
seq_lens,
|
||||
block_table,
|
||||
out_page_scratch,
|
||||
block_size,
|
||||
topk_indices_buffer[:num_decode_tokens, :topk_tokens],
|
||||
)
|
||||
|
||||
return topk_indices_buffer
|
||||
|
||||
Reference in New Issue
Block a user