[Distributed][MoonEP] BF16 integration of MoonEP balanced EP backend (#52101)

Signed-off-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
Co-authored-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
This commit is contained in:
Kevin Li
2026-09-18 10:34:01 -04:00
committed by GitHub
co-authored by kevinli
parent 2bdae2a5a8
commit 911548247a
15 changed files with 1338 additions and 12 deletions
@@ -35,6 +35,7 @@ from vllm.platforms import current_platform
from vllm.utils.import_utils import (
has_deep_ep,
has_deep_ep_v2,
has_moonep,
has_mori,
has_nixl_ep,
)
@@ -103,6 +104,13 @@ if current_platform.is_cuda_alike():
NIXL_EP_QUANT_BLOCK_SHAPE,
NixlEPPrepareAndFinalize,
)
if has_moonep():
from .prepare_finalize.moonep import (
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
MOONEP_DEFAULT_NUM_SMS,
MOONEP_DEFAULT_TOKEN_PADDING,
MoonEPPrepareAndFinalize,
)
def get_ep_all2all_manager(
@@ -279,6 +287,28 @@ def maybe_make_prepare_finalize(
sp_size=moe.moe_parallel_config.sp_size,
)
elif moe.use_moonep_kernels:
all_to_all_args = dict(
max_num_tokens_per_dp_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
num_topk=moe.experts_per_token,
num_global_experts=moe.num_experts,
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
token_padding=MOONEP_DEFAULT_TOKEN_PADDING,
num_sms=MOONEP_DEFAULT_NUM_SMS,
)
handle = all2all_manager.get_handle(all_to_all_args)
# The [E+B] weight layout is picked up from the experts (their
# process_weights_after_loading hook) once the layer has loaded and
# converted its weights.
prepare_finalize = MoonEPPrepareAndFinalize(
buffer_pool=handle,
max_tokens_per_rank=moe.max_num_tokens,
num_dispatchers=all2all_manager.world_size,
num_global_experts=moe.num_experts,
)
elif moe.use_mori_kernels:
assert quant_config is not None
@@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig:
def use_deepep_v2_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
@property
def use_moonep_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "moonep"
@staticmethod
def flatten_tp_across_dp_and_pcp(
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
@@ -1473,6 +1477,10 @@ class FusedMoEConfig:
def use_deepep_v2_kernels(self):
return self.moe_parallel_config.use_deepep_v2_kernels
@property
def use_moonep_kernels(self):
return self.moe_parallel_config.use_moonep_kernels
@property
def needs_round_robin_routing_tables(self):
return self.moe_parallel_config.needs_round_robin_routing_tables
@@ -0,0 +1,220 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations.
MoonEP dispatch delivers tokens already contiguous per weight row
(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute
is three grouped GEMMs over those segments with no permute/unpermute:
gate = grouped_mm(x, w_gate[E+B]) # [NvS, I]
up = grouped_mm(x, w_up[E+B]) # [NvS, I]
act = silu(gate) * up * route_weight # route weights applied here
out = grouped_mm(act, w_down[E+B]) # [NvS, H]
Rows ``[E, E+B)`` are the redundant-expert prefetch slots that
``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments
(including unused prefetch slots) are skipped by the grouped GEMM.
Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be
its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate
tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular
kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the
full layout is picked up from the layer in ``process_weights_after_loading``
and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
def moonep_grouped_gemm(
a: torch.Tensor,
w: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> torch.Tensor:
"""``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment.
Args:
a: ``[NvS, K]`` activations in segment order.
w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention).
cu_seqlens: ``[G]`` int32 cumulative segment end offsets.
Returns:
``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled.
"""
return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens)
class MoonEPExperts(mk.FusedMoEExpertsModular):
"""Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16)."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers)
assert quant_config.quant_dtype is None, (
"MoonEPExperts supports unquantized BF16 only"
)
if (
quant_config.gemm1_alpha is not None
or quant_config.gemm1_beta is not None
or quant_config.gemm1_clamp_limit is not None
):
raise NotImplementedError(
"MoonEPExperts implements plain silu(gate) * up only; SwiGLU "
"variants with alpha/beta/clamp parameters are not supported"
)
self._weight_layout = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# The [E+B] gate/up/down layout built by
# convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize
# reads it from here (post_init_setup) for weight prefetch.
layout = getattr(layer, "_moonep_weight_layout", None)
assert layout is not None, (
"MoonEPExperts requires the layer to carry _moonep_weight_layout "
"(set by convert_to_unquantized_kernel_format)"
)
self._weight_layout = layout
@property
def weight_layout(self):
return self._weight_layout
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# EPLB rearranges the named expert parameters as local_num_experts
# rows, which does not understand the replicated [E+B] layout.
return (
moe_parallel_config.use_moonep_kernels
and not moe_parallel_config.enable_eplb
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key is None and activation_key is None
def supports_chunking(self) -> bool:
# NvS is static and segments must stay whole.
return False
def supports_expert_map(self) -> bool:
# MoonEP addresses global expert rows directly.
return False
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def moe_problem_size(
self,
a1: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
# a1 is [NvS, H] in segment order, not token-major, so the base
# implementation's topk_ids/a1 row-count check does not apply.
assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3
num_rows, intermediate, hidden = w1.shape # [E+B, I, H]
assert a1.size(1) == hidden
return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# M is NvS. torch._grouped_mm allocates its own outputs, so no
# workspaces are needed.
return ((0,), (0,), (M, K))
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
# expert_map is ignored: MoonEP dispatches on global expert ids and
# the weights are addressed by global [E+B] row, on every rank.
assert activation == MoEActivation.SILU
assert self._weight_layout is not None, (
"process_weights_after_loading() not called"
)
assert expert_tokens_meta is not None
# MoonEPPrepareAndFinalize returns route weights in NvS order as the
# dispatched topk_weights, and per-row segment lengths as
# expert_tokens_meta.
route_weights_nvs = topk_weights
assert route_weights_nvs.dim() == 1
cu_seqlens = torch.cumsum(
expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32
)
assert cu_seqlens.numel() == w1.size(0)
gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens)
up = moonep_grouped_gemm(
hidden_states, self._weight_layout.full_up_weight, cu_seqlens
)
act = torch.nn.functional.silu(gate)
act.mul_(up)
if not apply_router_weight_on_input:
# Input-weighted routing (top-1, e.g. Llama 4) was already
# applied to the activations by prepare(), before dispatch.
act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1))
# Padding rows past the last segment come out zero-filled.
output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens))
@@ -40,6 +40,7 @@ class UnquantizedMoeBackend(Enum):
AITER = "ROCm AITER"
TRITON = "TRITON"
BATCHED_TRITON = "BATCHED_TRITON"
MOONEP = "MOONEP"
CPU = "CPU"
XPU = "XPU"
TPU = "TPU"
@@ -138,6 +139,13 @@ def backend_to_kernel_cls(
return [BatchedTritonExperts]
elif backend == UnquantizedMoeBackend.MOONEP:
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
MoonEPExperts,
)
return [MoonEPExperts]
elif backend == UnquantizedMoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
@@ -281,6 +289,13 @@ def select_unquantized_moe_backend(
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
# MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens),
# so it is not interchangeable with the token-major experts backends.
if moe_config.moe_parallel_config.use_moonep_kernels:
return _return_or_raise(
UnquantizedMoeBackend.MOONEP, moe_config, activation_format
)
runner_backend = moe_config.moe_backend
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
# modules_to_not_convert) falls through to auto instead of erroring.
@@ -366,7 +381,42 @@ def convert_to_unquantized_kernel_format(
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
layer: torch.nn.Module | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if unquantized_backend == UnquantizedMoeBackend.MOONEP:
if layer is None:
raise ValueError(
"MoonEP weight conversion requires the layer module to stash "
"the [E+B] weight layout on"
)
# MoonEP addresses expert weights by global [E+B] row and needs one
# contiguous tensor per projection (gate / up / down). The layer's
# w13_weight becomes the gate tensor and w2_weight the down tensor;
# the up tensor and the full layout are stashed on the layer for
# the kernel wiring in _setup_kernel.
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
gather_moonep_weight_layout,
)
if getattr(layer, "_moonep_weight_layout", None) is not None:
# On a reload the layer parameters were already replaced by the
# converted [E+B] gate/down tensors, so the source-format weights
# this conversion needs are gone (and the unregistered up
# projection cannot be recovered from them at all). Tracked in
# RFC #52095.
raise NotImplementedError(
"MoonEP does not support in-place weight reloads yet"
)
layout = gather_moonep_weight_layout(
w13_weight,
w2_weight,
num_global_experts=moe_config.num_experts,
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
)
layer._moonep_weight_layout = layout
return layout.full_gate_weight, layout.full_down_weight
if unquantized_backend == UnquantizedMoeBackend.AITER:
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
@@ -522,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
layer: torch.nn.Module | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
return convert_to_unquantized_kernel_format(
backend, moe_config, w13_weight, w2_weight
backend, moe_config, w13_weight, w2_weight, layer=layer
)
def make_kernel(
@@ -0,0 +1,429 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize.
BF16 correctness-first proof of concept on top of vLLM's modular kernel
interface.
MoonEP differs from DeepEP-style backends in two ways that shape this
integration:
- ``dispatch`` returns tokens already grouped by expert *row* (a fixed
``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together
with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is
no per-token topk id tensor after dispatch; the expert compute must be a
grouped GEMM over ``cu_seqlens`` segments.
- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert
prefetch slots. ``plan.experts_to_copy`` names the source expert of each
slot and ``Buffer.prefetch_weight`` must run between dispatch and expert
compute.
PoC limitations:
- BF16 / unquantized only, eager only.
- Expert weights are replicated in global-expert order on every rank
(memory-heavy). Production Kimi-K3 serving requires sharded
symmetric-memory expert ownership, where rows ``[0, E)`` physically alias
each home rank's parameter memory.
- Route weights are applied inside the expert compute and MoonEP's
``combine`` performs the K-sum, so ``finalize`` requires
``TopKWeightAndReduceNoOP``.
"""
from typing import Any, NamedTuple
import torch
import torch.nn.functional as F
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4
MOONEP_DEFAULT_TOKEN_PADDING = 128
MOONEP_DEFAULT_NUM_SMS = 32
# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128.
MOONEP_WEIGHT_TILE = 128
class MoonEPExpertWeightLayout(NamedTuple):
"""Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout.
Rows ``[0, E)`` hold expert weights in global expert order; rows
``[E, E+B)`` are mutable prefetch slots filled by
``Buffer.prefetch_weight``.
The prefetch slots never need re-zeroing between calls: the planner
marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by
``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so
stale slot contents are never read; used slots are fully overwritten.
"""
full_gate_weight: torch.Tensor
full_up_weight: torch.Tensor
full_down_weight: torch.Tensor
num_prefetch_slots: int
def make_moonep_weight_layout(
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
num_prefetch_slots: int,
) -> MoonEPExpertWeightLayout:
"""Build the replicated ``[E+B, ...]`` PoC weight layout.
``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight``
``[E, H, I]``, both BF16 in global expert order.
"""
if num_prefetch_slots <= 0:
raise ValueError(
f"num_prefetch_slots must be positive, got {num_prefetch_slots}"
)
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
raise NotImplementedError("MoonEP PoC supports BF16 weights only.")
num_experts, two_i, hidden_size = w13_weight.shape
intermediate_size = two_i // 2
if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size):
raise ValueError(
f"w2_weight shape {tuple(w2_weight.shape)} does not match "
f"w13_weight shape {tuple(w13_weight.shape)}"
)
if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE:
raise ValueError(
"MoonEP weight prefetch requires hidden_size and intermediate_size "
f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, "
f"I={intermediate_size}"
)
full_gate_weight = torch.empty(
num_experts + num_prefetch_slots,
intermediate_size,
hidden_size,
dtype=torch.bfloat16,
device=w13_weight.device,
)
full_up_weight = torch.empty_like(full_gate_weight)
full_down_weight = torch.empty(
num_experts + num_prefetch_slots,
hidden_size,
intermediate_size,
dtype=torch.bfloat16,
device=w2_weight.device,
)
full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :])
full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :])
full_down_weight[:num_experts].copy_(w2_weight)
full_gate_weight[num_experts:].zero_()
full_up_weight[num_experts:].zero_()
full_down_weight[num_experts:].zero_()
return MoonEPExpertWeightLayout(
full_gate_weight=full_gate_weight.contiguous(),
full_up_weight=full_up_weight.contiguous(),
full_down_weight=full_down_weight.contiguous(),
num_prefetch_slots=num_prefetch_slots,
)
def gather_moonep_weight_layout(
w13_local: torch.Tensor,
w2_local: torch.Tensor,
num_global_experts: int,
num_prefetch_slots: int,
) -> MoonEPExpertWeightLayout:
"""Build the replicated ``[E+B, ...]`` layout from this rank's local experts.
PoC bridge: each EP rank loads only its own experts (linear placement),
so all-gather them once at load time into global expert order on every
rank. Production MoonEP instead maps rows ``[0, E)`` onto each home
rank's parameter memory via symmetric memory (RFC #52095 item 5).
"""
from vllm.distributed import get_ep_group
ep_group = get_ep_group()
ep_size = ep_group.world_size
if ep_size == 1:
return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots)
if w13_local.size(0) * ep_size != num_global_experts:
raise NotImplementedError(
"MoonEP PoC requires num_experts to be evenly divisible across EP "
f"ranks: {num_global_experts} experts, {ep_size} ranks"
)
w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0)
w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0)
return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots)
MOONEP_MIN_DISPATCH_TOKENS = 128
class MoonEPBufferPool:
"""Lazily creates ``moonep.Buffer`` instances at power-of-two capacities.
``Buffer`` fixes its token capacity ``S`` at construction, so a single
full-capacity buffer would pad every dispatch to
``max_num_batched_tokens``. The pool instead serves the smallest
power-of-two capacity that fits the step's token count, so decode-sized
batches dispatch a few hundred slots rather than the full capacity.
Selection must be identical on every EP rank (dispatch is collective and
``Buffer`` construction is itself a collective): callers derive the token
count from ``dp_metadata`` (the max across DP ranks) so all ranks create
and pick the same buffer at the same step.
"""
def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int):
self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"}
self.max_tokens_per_rank = max_tokens_per_rank
self._buffers: dict[int, Any] = {}
def capacity_for(self, num_tokens: int) -> int:
if num_tokens >= self.max_tokens_per_rank:
return self.max_tokens_per_rank
cap = 1 << max(num_tokens - 1, 0).bit_length()
return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank)
def get(self, num_tokens: int) -> tuple[int, Any]:
"""Return ``(capacity, buffer)`` for the given step token count."""
from moonep import Buffer # type: ignore[import-not-found]
capacity = self.capacity_for(num_tokens)
buffer = self._buffers.get(capacity)
if buffer is None:
buffer = Buffer(S=capacity, **self._kwargs)
self._buffers[capacity] = buffer
return capacity, buffer
def destroy(self) -> None:
for buffer in self._buffers.values():
buffer.destroy()
self._buffers.clear()
class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""Prepare/Finalize using MoonEP balanced dispatch/combine.
``prepare`` pads the batch to the selected buffer's static token
capacity, dispatches, runs ``prefetch_weight`` for the planned redundant
experts, and stashes the ``plan`` for ``finalize`` (the same pattern
DeepEP-HT uses for its handle). Downstream expert compute must consume
the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``.
"""
def __init__(
self,
buffer_pool: "MoonEPBufferPool",
max_tokens_per_rank: int,
num_dispatchers: int,
num_global_experts: int,
weight_layout: MoonEPExpertWeightLayout | None = None,
):
super().__init__()
self.buffer_pool = buffer_pool
self.max_tokens_per_rank = max_tokens_per_rank
self.num_dispatchers_ = num_dispatchers
self.num_global_experts = num_global_experts
self.weight_layout = weight_layout
self._fused_experts: Any = None
# dispatch state consumed by finalize (and the expert runner)
self._plan: Any = None
self._cu_seqlens: torch.Tensor | None = None
self._num_tokens: int = 0
self._buffer: Any = None # the pool buffer used by the current step
def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None:
# The [E+B] weight layout is attached to the experts by their
# process_weights_after_loading hook (after this runs), so keep the
# reference and resolve the layout lazily in prepare().
self._fused_experts = fused_experts
def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout:
if self.weight_layout is None and self._fused_experts is not None:
layout = getattr(self._fused_experts, "weight_layout", None)
if isinstance(layout, MoonEPExpertWeightLayout):
self.weight_layout = layout
# Redundant experts' weights must be in rows [E, E+B) before the
# expert compute reads them; skipping prefetch silently corrupts
# output.
assert self.weight_layout is not None, (
"MoonEPPrepareAndFinalize: weight layout not available (the "
"experts' process_weights_after_loading has not run)"
)
return self.weight_layout
@property
def num_dispatched_slots(self) -> int:
"""``NvS`` of the buffer selected by the current step's prepare()."""
assert self._buffer is not None, "prepare() has not been called"
return int(self._buffer._ctx["NvS"])
def num_dispatchers(self) -> int:
return self.num_dispatchers_
def output_is_reduced(self) -> bool:
# combine returns the fully weighted+reduced token-major output
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
def max_num_tokens_per_rank(self) -> int | None:
return self.max_tokens_per_rank
def topk_indices_dtype(self) -> torch.dtype | None:
return torch.int32
def supports_async(self) -> bool:
return False
@property
def cu_seqlens(self) -> torch.Tensor:
assert self._cu_seqlens is not None, "prepare() has not been called"
return self._cu_seqlens
@property
def plan(self) -> Any:
assert self._plan is not None, "prepare() has not been called"
return self._plan
def _pad_to_capacity(
self,
a1: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
capacity: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
num_tokens = a1.shape[0]
if num_tokens > capacity:
raise ValueError(
f"MoonEP static buffer capacity exceeded: {num_tokens} tokens "
f"> S={capacity}."
)
a1 = a1.contiguous()
topk_ids = topk_ids.to(dtype=torch.int32)
topk_weights = topk_weights.to(dtype=torch.float32)
# MoonEP's planner has no sentinel for invalid expert ids, so map
# negative ids (padded/invalid slots from the engine) to expert 0
# with zero route weight — the same convention as the capacity
# padding rows below. Wasted compute, never wrong output.
invalid = topk_ids < 0
topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids)
topk_weights = torch.where(
invalid, torch.zeros_like(topk_weights), topk_weights
).contiguous()
topk_ids = topk_ids.contiguous()
if num_tokens == capacity:
return a1, topk_ids, topk_weights, num_tokens
pad = capacity - num_tokens
return (
F.pad(a1, (0, 0, 0, pad)),
F.pad(topk_ids, (0, 0, 0, pad)),
F.pad(topk_weights, (0, 0, 0, pad)),
num_tokens,
)
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
if a1.dtype != torch.bfloat16:
raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.")
if quant_config.quant_dtype is not None:
raise NotImplementedError("MoonEP PoC does not support quantized dispatch.")
assert num_experts == self.num_global_experts
assert self._plan is None, (
"MoonEPPrepareAndFinalize.prepare() called again before finalize()"
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input requires top-1 routing"
)
a1 = a1 * topk_weights.to(a1.dtype)
# Buffer choice must be rank-symmetric: use the max token count
# across DP ranks (every rank sees the same dp_metadata), falling
# back to the local count outside a forward context (tests).
num_tokens = a1.shape[0]
try:
dp_meta = get_forward_context().dp_metadata
except AssertionError:
dp_meta = None
if dp_meta is not None:
num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max())
capacity, self._buffer = self.buffer_pool.get(num_tokens)
a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity(
a1, topk_ids, topk_weights, capacity
)
tokens_per_expert = torch.bincount(
topk_ids.reshape(-1).to(dtype=torch.int64),
minlength=num_experts,
).to(dtype=torch.int32)
hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch(
a1,
topk_weights,
topk_ids,
tokens_per_expert,
)
self._plan = plan
self._cu_seqlens = cu_seqlens
weight_layout = self._resolve_weight_layout()
self._buffer.prefetch_weight(
plan=plan,
full_gate_weight=weight_layout.full_gate_weight,
full_up_weight=weight_layout.full_up_weight,
full_down_weight=weight_layout.full_down_weight,
)
# Segment sizes per [E+B] weight row. NOTE: this is per *row*, not
# per local expert, and NvS rows are expert-grouped rather than
# token-major — only MoonEP-aware expert implementations can consume
# this activation layout.
expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1))
expert_tokens_meta = mk.ExpertTokensMetadata(
expert_num_tokens=expert_num_tokens,
expert_num_tokens_cpu=None,
)
# MoonEP has no post-dispatch per-token topk ids/weights; route
# weights come back in NvS order.
return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs
def finalize(
self,
output: torch.Tensor,
fused_expert_output: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> None:
# Route weights are applied in the expert compute and MoonEP's
# combine performs the K-sum, so any weight/reduce work in finalize
# means MoonEP was paired with the wrong kind of experts.
assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), (
"MoonEP requires TopKWeightAndReduceNoOP, got "
f"{type(weight_and_reduce_impl).__name__}"
)
combined, _, _ = self._buffer.combine(
plan=self.plan,
hidden_nvsh=fused_expert_output,
route_weights_nvs=None,
)
output.copy_(combined[: self._num_tokens])
self._plan = None
self._cu_seqlens = None
@@ -49,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# --8<-- [end:unquantized_fused_moe]
supports_pre_processed_weights = True
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
moe_config=self.moe,
)
# MoonEP replaces the named parameters with its [E+B] gate/down
# split and keeps the up projection outside the parameter registry,
# so a pre-processed (e.g. ipc_cache) load cannot reconstruct it.
self.supports_pre_processed_weights = (
self.unquantized_backend != UnquantizedMoeBackend.MOONEP
)
@property
def supports_eplb(self) -> bool:
@@ -171,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
moe_config=layer.moe_config,
w13_weight=w13,
w2_weight=w2,
layer=layer,
)
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
# On the first call we replace the parameter normally. On subsequent
@@ -191,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# does not need to be re-built.
self._init_moe_kernel(layer)
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
# The CPU experts need the layer itself for the setup that
# convert_to_unquantized_kernel_format cannot express, since
# it only sees the two weight tensors: padding and prepacking
# into the grouped-gemm layout (bias included), and capturing
# the router config that monolithic apply() cannot carry.
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
# No-op by default. Experts that need the layer itself for setup
# convert_to_unquantized_kernel_format cannot express override
# it: CPU prepacks into its grouped-gemm layout and captures the
# router config; MoonEP picks up the [E+B] weight layout for
# prefetch and the up projection.
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
"""Build the MoE kernel from the layer's current (shuffled) weights."""