[Distributed][MoonEP] BF16 integration of MoonEP balanced EP backend (#52101)
Signed-off-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com> Co-authored-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
This commit is contained in:
@@ -35,6 +35,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import (
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_moonep,
|
||||
has_mori,
|
||||
has_nixl_ep,
|
||||
)
|
||||
@@ -103,6 +104,13 @@ if current_platform.is_cuda_alike():
|
||||
NIXL_EP_QUANT_BLOCK_SHAPE,
|
||||
NixlEPPrepareAndFinalize,
|
||||
)
|
||||
if has_moonep():
|
||||
from .prepare_finalize.moonep import (
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
MOONEP_DEFAULT_NUM_SMS,
|
||||
MOONEP_DEFAULT_TOKEN_PADDING,
|
||||
MoonEPPrepareAndFinalize,
|
||||
)
|
||||
|
||||
|
||||
def get_ep_all2all_manager(
|
||||
@@ -279,6 +287,28 @@ def maybe_make_prepare_finalize(
|
||||
sp_size=moe.moe_parallel_config.sp_size,
|
||||
)
|
||||
|
||||
elif moe.use_moonep_kernels:
|
||||
all_to_all_args = dict(
|
||||
max_num_tokens_per_dp_rank=moe.max_num_tokens,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
num_topk=moe.experts_per_token,
|
||||
num_global_experts=moe.num_experts,
|
||||
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
token_padding=MOONEP_DEFAULT_TOKEN_PADDING,
|
||||
num_sms=MOONEP_DEFAULT_NUM_SMS,
|
||||
)
|
||||
handle = all2all_manager.get_handle(all_to_all_args)
|
||||
|
||||
# The [E+B] weight layout is picked up from the experts (their
|
||||
# process_weights_after_loading hook) once the layer has loaded and
|
||||
# converted its weights.
|
||||
prepare_finalize = MoonEPPrepareAndFinalize(
|
||||
buffer_pool=handle,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
num_global_experts=moe.num_experts,
|
||||
)
|
||||
|
||||
elif moe.use_mori_kernels:
|
||||
assert quant_config is not None
|
||||
|
||||
|
||||
@@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig:
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
|
||||
|
||||
@property
|
||||
def use_moonep_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "moonep"
|
||||
|
||||
@staticmethod
|
||||
def flatten_tp_across_dp_and_pcp(
|
||||
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
|
||||
@@ -1473,6 +1477,10 @@ class FusedMoEConfig:
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_v2_kernels
|
||||
|
||||
@property
|
||||
def use_moonep_kernels(self):
|
||||
return self.moe_parallel_config.use_moonep_kernels
|
||||
|
||||
@property
|
||||
def needs_round_robin_routing_tables(self):
|
||||
return self.moe_parallel_config.needs_round_robin_routing_tables
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations.
|
||||
|
||||
MoonEP dispatch delivers tokens already contiguous per weight row
|
||||
(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute
|
||||
is three grouped GEMMs over those segments with no permute/unpermute:
|
||||
|
||||
gate = grouped_mm(x, w_gate[E+B]) # [NvS, I]
|
||||
up = grouped_mm(x, w_up[E+B]) # [NvS, I]
|
||||
act = silu(gate) * up * route_weight # route weights applied here
|
||||
out = grouped_mm(act, w_down[E+B]) # [NvS, H]
|
||||
|
||||
Rows ``[E, E+B)`` are the redundant-expert prefetch slots that
|
||||
``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments
|
||||
(including unused prefetch slots) are skipped by the grouped GEMM.
|
||||
|
||||
Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be
|
||||
its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate
|
||||
tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular
|
||||
kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the
|
||||
full layout is picked up from the layer in ``process_weights_after_loading``
|
||||
and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def moonep_grouped_gemm(
|
||||
a: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment.
|
||||
|
||||
Args:
|
||||
a: ``[NvS, K]`` activations in segment order.
|
||||
w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention).
|
||||
cu_seqlens: ``[G]`` int32 cumulative segment end offsets.
|
||||
|
||||
Returns:
|
||||
``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled.
|
||||
|
||||
"""
|
||||
return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens)
|
||||
|
||||
|
||||
class MoonEPExperts(mk.FusedMoEExpertsModular):
|
||||
"""Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers)
|
||||
assert quant_config.quant_dtype is None, (
|
||||
"MoonEPExperts supports unquantized BF16 only"
|
||||
)
|
||||
if (
|
||||
quant_config.gemm1_alpha is not None
|
||||
or quant_config.gemm1_beta is not None
|
||||
or quant_config.gemm1_clamp_limit is not None
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MoonEPExperts implements plain silu(gate) * up only; SwiGLU "
|
||||
"variants with alpha/beta/clamp parameters are not supported"
|
||||
)
|
||||
self._weight_layout = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# The [E+B] gate/up/down layout built by
|
||||
# convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize
|
||||
# reads it from here (post_init_setup) for weight prefetch.
|
||||
layout = getattr(layer, "_moonep_weight_layout", None)
|
||||
assert layout is not None, (
|
||||
"MoonEPExperts requires the layer to carry _moonep_weight_layout "
|
||||
"(set by convert_to_unquantized_kernel_format)"
|
||||
)
|
||||
self._weight_layout = layout
|
||||
|
||||
@property
|
||||
def weight_layout(self):
|
||||
return self._weight_layout
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SILU
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# EPLB rearranges the named expert parameters as local_num_experts
|
||||
# rows, which does not understand the replicated [E+B] layout.
|
||||
return (
|
||||
moe_parallel_config.use_moonep_kernels
|
||||
and not moe_parallel_config.enable_eplb
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key is None and activation_key is None
|
||||
|
||||
def supports_chunking(self) -> bool:
|
||||
# NvS is static and segments must stay whole.
|
||||
return False
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
# MoonEP addresses global expert rows directly.
|
||||
return False
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
# a1 is [NvS, H] in segment order, not token-major, so the base
|
||||
# implementation's topk_ids/a1 row-count check does not apply.
|
||||
assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3
|
||||
num_rows, intermediate, hidden = w1.shape # [E+B, I, H]
|
||||
assert a1.size(1) == hidden
|
||||
return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# M is NvS. torch._grouped_mm allocates its own outputs, so no
|
||||
# workspaces are needed.
|
||||
return ((0,), (0,), (M, K))
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
# expert_map is ignored: MoonEP dispatches on global expert ids and
|
||||
# the weights are addressed by global [E+B] row, on every rank.
|
||||
assert activation == MoEActivation.SILU
|
||||
assert self._weight_layout is not None, (
|
||||
"process_weights_after_loading() not called"
|
||||
)
|
||||
assert expert_tokens_meta is not None
|
||||
# MoonEPPrepareAndFinalize returns route weights in NvS order as the
|
||||
# dispatched topk_weights, and per-row segment lengths as
|
||||
# expert_tokens_meta.
|
||||
route_weights_nvs = topk_weights
|
||||
assert route_weights_nvs.dim() == 1
|
||||
cu_seqlens = torch.cumsum(
|
||||
expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32
|
||||
)
|
||||
|
||||
assert cu_seqlens.numel() == w1.size(0)
|
||||
|
||||
gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens)
|
||||
up = moonep_grouped_gemm(
|
||||
hidden_states, self._weight_layout.full_up_weight, cu_seqlens
|
||||
)
|
||||
act = torch.nn.functional.silu(gate)
|
||||
act.mul_(up)
|
||||
if not apply_router_weight_on_input:
|
||||
# Input-weighted routing (top-1, e.g. Llama 4) was already
|
||||
# applied to the activations by prepare(), before dispatch.
|
||||
act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1))
|
||||
# Padding rows past the last segment come out zero-filled.
|
||||
output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens))
|
||||
@@ -40,6 +40,7 @@ class UnquantizedMoeBackend(Enum):
|
||||
AITER = "ROCm AITER"
|
||||
TRITON = "TRITON"
|
||||
BATCHED_TRITON = "BATCHED_TRITON"
|
||||
MOONEP = "MOONEP"
|
||||
CPU = "CPU"
|
||||
XPU = "XPU"
|
||||
TPU = "TPU"
|
||||
@@ -138,6 +139,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [BatchedTritonExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.MOONEP:
|
||||
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
|
||||
MoonEPExperts,
|
||||
)
|
||||
|
||||
return [MoonEPExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
|
||||
|
||||
@@ -281,6 +289,13 @@ def select_unquantized_moe_backend(
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens),
|
||||
# so it is not interchangeable with the token-major experts backends.
|
||||
if moe_config.moe_parallel_config.use_moonep_kernels:
|
||||
return _return_or_raise(
|
||||
UnquantizedMoeBackend.MOONEP, moe_config, activation_format
|
||||
)
|
||||
|
||||
runner_backend = moe_config.moe_backend
|
||||
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
|
||||
# modules_to_not_convert) falls through to auto instead of erroring.
|
||||
@@ -366,7 +381,42 @@ def convert_to_unquantized_kernel_format(
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if unquantized_backend == UnquantizedMoeBackend.MOONEP:
|
||||
if layer is None:
|
||||
raise ValueError(
|
||||
"MoonEP weight conversion requires the layer module to stash "
|
||||
"the [E+B] weight layout on"
|
||||
)
|
||||
# MoonEP addresses expert weights by global [E+B] row and needs one
|
||||
# contiguous tensor per projection (gate / up / down). The layer's
|
||||
# w13_weight becomes the gate tensor and w2_weight the down tensor;
|
||||
# the up tensor and the full layout are stashed on the layer for
|
||||
# the kernel wiring in _setup_kernel.
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
gather_moonep_weight_layout,
|
||||
)
|
||||
|
||||
if getattr(layer, "_moonep_weight_layout", None) is not None:
|
||||
# On a reload the layer parameters were already replaced by the
|
||||
# converted [E+B] gate/down tensors, so the source-format weights
|
||||
# this conversion needs are gone (and the unregistered up
|
||||
# projection cannot be recovered from them at all). Tracked in
|
||||
# RFC #52095.
|
||||
raise NotImplementedError(
|
||||
"MoonEP does not support in-place weight reloads yet"
|
||||
)
|
||||
layout = gather_moonep_weight_layout(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
num_global_experts=moe_config.num_experts,
|
||||
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
)
|
||||
layer._moonep_weight_layout = layout
|
||||
return layout.full_gate_weight, layout.full_down_weight
|
||||
|
||||
if unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
w13_weight.is_shuffled = True
|
||||
@@ -522,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return convert_to_unquantized_kernel_format(
|
||||
backend, moe_config, w13_weight, w2_weight
|
||||
backend, moe_config, w13_weight, w2_weight, layer=layer
|
||||
)
|
||||
|
||||
def make_kernel(
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize.
|
||||
|
||||
BF16 correctness-first proof of concept on top of vLLM's modular kernel
|
||||
interface.
|
||||
|
||||
MoonEP differs from DeepEP-style backends in two ways that shape this
|
||||
integration:
|
||||
|
||||
- ``dispatch`` returns tokens already grouped by expert *row* (a fixed
|
||||
``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together
|
||||
with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is
|
||||
no per-token topk id tensor after dispatch; the expert compute must be a
|
||||
grouped GEMM over ``cu_seqlens`` segments.
|
||||
- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert
|
||||
prefetch slots. ``plan.experts_to_copy`` names the source expert of each
|
||||
slot and ``Buffer.prefetch_weight`` must run between dispatch and expert
|
||||
compute.
|
||||
|
||||
PoC limitations:
|
||||
- BF16 / unquantized only, eager only.
|
||||
- Expert weights are replicated in global-expert order on every rank
|
||||
(memory-heavy). Production Kimi-K3 serving requires sharded
|
||||
symmetric-memory expert ownership, where rows ``[0, E)`` physically alias
|
||||
each home rank's parameter memory.
|
||||
- Route weights are applied inside the expert compute and MoonEP's
|
||||
``combine`` performs the K-sum, so ``finalize`` requires
|
||||
``TopKWeightAndReduceNoOP``.
|
||||
"""
|
||||
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4
|
||||
MOONEP_DEFAULT_TOKEN_PADDING = 128
|
||||
MOONEP_DEFAULT_NUM_SMS = 32
|
||||
# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128.
|
||||
MOONEP_WEIGHT_TILE = 128
|
||||
|
||||
|
||||
class MoonEPExpertWeightLayout(NamedTuple):
|
||||
"""Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout.
|
||||
|
||||
Rows ``[0, E)`` hold expert weights in global expert order; rows
|
||||
``[E, E+B)`` are mutable prefetch slots filled by
|
||||
``Buffer.prefetch_weight``.
|
||||
|
||||
The prefetch slots never need re-zeroing between calls: the planner
|
||||
marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by
|
||||
``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so
|
||||
stale slot contents are never read; used slots are fully overwritten.
|
||||
"""
|
||||
|
||||
full_gate_weight: torch.Tensor
|
||||
full_up_weight: torch.Tensor
|
||||
full_down_weight: torch.Tensor
|
||||
num_prefetch_slots: int
|
||||
|
||||
|
||||
def make_moonep_weight_layout(
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
num_prefetch_slots: int,
|
||||
) -> MoonEPExpertWeightLayout:
|
||||
"""Build the replicated ``[E+B, ...]`` PoC weight layout.
|
||||
|
||||
``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight``
|
||||
``[E, H, I]``, both BF16 in global expert order.
|
||||
"""
|
||||
if num_prefetch_slots <= 0:
|
||||
raise ValueError(
|
||||
f"num_prefetch_slots must be positive, got {num_prefetch_slots}"
|
||||
)
|
||||
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("MoonEP PoC supports BF16 weights only.")
|
||||
num_experts, two_i, hidden_size = w13_weight.shape
|
||||
intermediate_size = two_i // 2
|
||||
if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size):
|
||||
raise ValueError(
|
||||
f"w2_weight shape {tuple(w2_weight.shape)} does not match "
|
||||
f"w13_weight shape {tuple(w13_weight.shape)}"
|
||||
)
|
||||
if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE:
|
||||
raise ValueError(
|
||||
"MoonEP weight prefetch requires hidden_size and intermediate_size "
|
||||
f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, "
|
||||
f"I={intermediate_size}"
|
||||
)
|
||||
|
||||
full_gate_weight = torch.empty(
|
||||
num_experts + num_prefetch_slots,
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=w13_weight.device,
|
||||
)
|
||||
full_up_weight = torch.empty_like(full_gate_weight)
|
||||
full_down_weight = torch.empty(
|
||||
num_experts + num_prefetch_slots,
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=w2_weight.device,
|
||||
)
|
||||
|
||||
full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :])
|
||||
full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :])
|
||||
full_down_weight[:num_experts].copy_(w2_weight)
|
||||
full_gate_weight[num_experts:].zero_()
|
||||
full_up_weight[num_experts:].zero_()
|
||||
full_down_weight[num_experts:].zero_()
|
||||
|
||||
return MoonEPExpertWeightLayout(
|
||||
full_gate_weight=full_gate_weight.contiguous(),
|
||||
full_up_weight=full_up_weight.contiguous(),
|
||||
full_down_weight=full_down_weight.contiguous(),
|
||||
num_prefetch_slots=num_prefetch_slots,
|
||||
)
|
||||
|
||||
|
||||
def gather_moonep_weight_layout(
|
||||
w13_local: torch.Tensor,
|
||||
w2_local: torch.Tensor,
|
||||
num_global_experts: int,
|
||||
num_prefetch_slots: int,
|
||||
) -> MoonEPExpertWeightLayout:
|
||||
"""Build the replicated ``[E+B, ...]`` layout from this rank's local experts.
|
||||
|
||||
PoC bridge: each EP rank loads only its own experts (linear placement),
|
||||
so all-gather them once at load time into global expert order on every
|
||||
rank. Production MoonEP instead maps rows ``[0, E)`` onto each home
|
||||
rank's parameter memory via symmetric memory (RFC #52095 item 5).
|
||||
"""
|
||||
from vllm.distributed import get_ep_group
|
||||
|
||||
ep_group = get_ep_group()
|
||||
ep_size = ep_group.world_size
|
||||
if ep_size == 1:
|
||||
return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots)
|
||||
if w13_local.size(0) * ep_size != num_global_experts:
|
||||
raise NotImplementedError(
|
||||
"MoonEP PoC requires num_experts to be evenly divisible across EP "
|
||||
f"ranks: {num_global_experts} experts, {ep_size} ranks"
|
||||
)
|
||||
w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0)
|
||||
w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0)
|
||||
return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots)
|
||||
|
||||
|
||||
MOONEP_MIN_DISPATCH_TOKENS = 128
|
||||
|
||||
|
||||
class MoonEPBufferPool:
|
||||
"""Lazily creates ``moonep.Buffer`` instances at power-of-two capacities.
|
||||
|
||||
``Buffer`` fixes its token capacity ``S`` at construction, so a single
|
||||
full-capacity buffer would pad every dispatch to
|
||||
``max_num_batched_tokens``. The pool instead serves the smallest
|
||||
power-of-two capacity that fits the step's token count, so decode-sized
|
||||
batches dispatch a few hundred slots rather than the full capacity.
|
||||
|
||||
Selection must be identical on every EP rank (dispatch is collective and
|
||||
``Buffer`` construction is itself a collective): callers derive the token
|
||||
count from ``dp_metadata`` (the max across DP ranks) so all ranks create
|
||||
and pick the same buffer at the same step.
|
||||
"""
|
||||
|
||||
def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int):
|
||||
self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"}
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self._buffers: dict[int, Any] = {}
|
||||
|
||||
def capacity_for(self, num_tokens: int) -> int:
|
||||
if num_tokens >= self.max_tokens_per_rank:
|
||||
return self.max_tokens_per_rank
|
||||
cap = 1 << max(num_tokens - 1, 0).bit_length()
|
||||
return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank)
|
||||
|
||||
def get(self, num_tokens: int) -> tuple[int, Any]:
|
||||
"""Return ``(capacity, buffer)`` for the given step token count."""
|
||||
from moonep import Buffer # type: ignore[import-not-found]
|
||||
|
||||
capacity = self.capacity_for(num_tokens)
|
||||
buffer = self._buffers.get(capacity)
|
||||
if buffer is None:
|
||||
buffer = Buffer(S=capacity, **self._kwargs)
|
||||
self._buffers[capacity] = buffer
|
||||
return capacity, buffer
|
||||
|
||||
def destroy(self) -> None:
|
||||
for buffer in self._buffers.values():
|
||||
buffer.destroy()
|
||||
self._buffers.clear()
|
||||
|
||||
|
||||
class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""Prepare/Finalize using MoonEP balanced dispatch/combine.
|
||||
|
||||
``prepare`` pads the batch to the selected buffer's static token
|
||||
capacity, dispatches, runs ``prefetch_weight`` for the planned redundant
|
||||
experts, and stashes the ``plan`` for ``finalize`` (the same pattern
|
||||
DeepEP-HT uses for its handle). Downstream expert compute must consume
|
||||
the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
buffer_pool: "MoonEPBufferPool",
|
||||
max_tokens_per_rank: int,
|
||||
num_dispatchers: int,
|
||||
num_global_experts: int,
|
||||
weight_layout: MoonEPExpertWeightLayout | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.buffer_pool = buffer_pool
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.num_global_experts = num_global_experts
|
||||
self.weight_layout = weight_layout
|
||||
self._fused_experts: Any = None
|
||||
|
||||
# dispatch state consumed by finalize (and the expert runner)
|
||||
self._plan: Any = None
|
||||
self._cu_seqlens: torch.Tensor | None = None
|
||||
self._num_tokens: int = 0
|
||||
self._buffer: Any = None # the pool buffer used by the current step
|
||||
|
||||
def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None:
|
||||
# The [E+B] weight layout is attached to the experts by their
|
||||
# process_weights_after_loading hook (after this runs), so keep the
|
||||
# reference and resolve the layout lazily in prepare().
|
||||
self._fused_experts = fused_experts
|
||||
|
||||
def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout:
|
||||
if self.weight_layout is None and self._fused_experts is not None:
|
||||
layout = getattr(self._fused_experts, "weight_layout", None)
|
||||
if isinstance(layout, MoonEPExpertWeightLayout):
|
||||
self.weight_layout = layout
|
||||
# Redundant experts' weights must be in rows [E, E+B) before the
|
||||
# expert compute reads them; skipping prefetch silently corrupts
|
||||
# output.
|
||||
assert self.weight_layout is not None, (
|
||||
"MoonEPPrepareAndFinalize: weight layout not available (the "
|
||||
"experts' process_weights_after_loading has not run)"
|
||||
)
|
||||
return self.weight_layout
|
||||
|
||||
@property
|
||||
def num_dispatched_slots(self) -> int:
|
||||
"""``NvS`` of the buffer selected by the current step's prepare()."""
|
||||
assert self._buffer is not None, "prepare() has not been called"
|
||||
return int(self._buffer._ctx["NvS"])
|
||||
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
# combine returns the fully weighted+reduced token-major output
|
||||
return True
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
return self.max_tokens_per_rank
|
||||
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
return torch.int32
|
||||
|
||||
def supports_async(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def cu_seqlens(self) -> torch.Tensor:
|
||||
assert self._cu_seqlens is not None, "prepare() has not been called"
|
||||
return self._cu_seqlens
|
||||
|
||||
@property
|
||||
def plan(self) -> Any:
|
||||
assert self._plan is not None, "prepare() has not been called"
|
||||
return self._plan
|
||||
|
||||
def _pad_to_capacity(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
capacity: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
num_tokens = a1.shape[0]
|
||||
if num_tokens > capacity:
|
||||
raise ValueError(
|
||||
f"MoonEP static buffer capacity exceeded: {num_tokens} tokens "
|
||||
f"> S={capacity}."
|
||||
)
|
||||
a1 = a1.contiguous()
|
||||
topk_ids = topk_ids.to(dtype=torch.int32)
|
||||
topk_weights = topk_weights.to(dtype=torch.float32)
|
||||
# MoonEP's planner has no sentinel for invalid expert ids, so map
|
||||
# negative ids (padded/invalid slots from the engine) to expert 0
|
||||
# with zero route weight — the same convention as the capacity
|
||||
# padding rows below. Wasted compute, never wrong output.
|
||||
invalid = topk_ids < 0
|
||||
topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids)
|
||||
topk_weights = torch.where(
|
||||
invalid, torch.zeros_like(topk_weights), topk_weights
|
||||
).contiguous()
|
||||
topk_ids = topk_ids.contiguous()
|
||||
if num_tokens == capacity:
|
||||
return a1, topk_ids, topk_weights, num_tokens
|
||||
pad = capacity - num_tokens
|
||||
return (
|
||||
F.pad(a1, (0, 0, 0, pad)),
|
||||
F.pad(topk_ids, (0, 0, 0, pad)),
|
||||
F.pad(topk_weights, (0, 0, 0, pad)),
|
||||
num_tokens,
|
||||
)
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
if a1.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.")
|
||||
if quant_config.quant_dtype is not None:
|
||||
raise NotImplementedError("MoonEP PoC does not support quantized dispatch.")
|
||||
assert num_experts == self.num_global_experts
|
||||
assert self._plan is None, (
|
||||
"MoonEPPrepareAndFinalize.prepare() called again before finalize()"
|
||||
)
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input requires top-1 routing"
|
||||
)
|
||||
a1 = a1 * topk_weights.to(a1.dtype)
|
||||
|
||||
# Buffer choice must be rank-symmetric: use the max token count
|
||||
# across DP ranks (every rank sees the same dp_metadata), falling
|
||||
# back to the local count outside a forward context (tests).
|
||||
num_tokens = a1.shape[0]
|
||||
try:
|
||||
dp_meta = get_forward_context().dp_metadata
|
||||
except AssertionError:
|
||||
dp_meta = None
|
||||
if dp_meta is not None:
|
||||
num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max())
|
||||
capacity, self._buffer = self.buffer_pool.get(num_tokens)
|
||||
|
||||
a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity(
|
||||
a1, topk_ids, topk_weights, capacity
|
||||
)
|
||||
tokens_per_expert = torch.bincount(
|
||||
topk_ids.reshape(-1).to(dtype=torch.int64),
|
||||
minlength=num_experts,
|
||||
).to(dtype=torch.int32)
|
||||
|
||||
hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch(
|
||||
a1,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
tokens_per_expert,
|
||||
)
|
||||
self._plan = plan
|
||||
self._cu_seqlens = cu_seqlens
|
||||
|
||||
weight_layout = self._resolve_weight_layout()
|
||||
self._buffer.prefetch_weight(
|
||||
plan=plan,
|
||||
full_gate_weight=weight_layout.full_gate_weight,
|
||||
full_up_weight=weight_layout.full_up_weight,
|
||||
full_down_weight=weight_layout.full_down_weight,
|
||||
)
|
||||
|
||||
# Segment sizes per [E+B] weight row. NOTE: this is per *row*, not
|
||||
# per local expert, and NvS rows are expert-grouped rather than
|
||||
# token-major — only MoonEP-aware expert implementations can consume
|
||||
# this activation layout.
|
||||
expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1))
|
||||
expert_tokens_meta = mk.ExpertTokensMetadata(
|
||||
expert_num_tokens=expert_num_tokens,
|
||||
expert_num_tokens_cpu=None,
|
||||
)
|
||||
|
||||
# MoonEP has no post-dispatch per-token topk ids/weights; route
|
||||
# weights come back in NvS order.
|
||||
return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
# Route weights are applied in the expert compute and MoonEP's
|
||||
# combine performs the K-sum, so any weight/reduce work in finalize
|
||||
# means MoonEP was paired with the wrong kind of experts.
|
||||
assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), (
|
||||
"MoonEP requires TopKWeightAndReduceNoOP, got "
|
||||
f"{type(weight_and_reduce_impl).__name__}"
|
||||
)
|
||||
combined, _, _ = self._buffer.combine(
|
||||
plan=self.plan,
|
||||
hidden_nvsh=fused_expert_output,
|
||||
route_weights_nvs=None,
|
||||
)
|
||||
output.copy_(combined[: self._num_tokens])
|
||||
self._plan = None
|
||||
self._cu_seqlens = None
|
||||
@@ -49,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
# --8<-- [end:unquantized_fused_moe]
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=self.moe,
|
||||
)
|
||||
# MoonEP replaces the named parameters with its [E+B] gate/down
|
||||
# split and keeps the up projection outside the parameter registry,
|
||||
# so a pre-processed (e.g. ipc_cache) load cannot reconstruct it.
|
||||
self.supports_pre_processed_weights = (
|
||||
self.unquantized_backend != UnquantizedMoeBackend.MOONEP
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
@@ -171,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
moe_config=layer.moe_config,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
layer=layer,
|
||||
)
|
||||
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
|
||||
# On the first call we replace the parameter normally. On subsequent
|
||||
@@ -191,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# does not need to be re-built.
|
||||
self._init_moe_kernel(layer)
|
||||
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
# The CPU experts need the layer itself for the setup that
|
||||
# convert_to_unquantized_kernel_format cannot express, since
|
||||
# it only sees the two weight tensors: padding and prepacking
|
||||
# into the grouped-gemm layout (bias included), and capturing
|
||||
# the router config that monolithic apply() cannot carry.
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
# No-op by default. Experts that need the layer itself for setup
|
||||
# convert_to_unquantized_kernel_format cannot express override
|
||||
# it: CPU prepacks into its grouped-gemm layout and captures the
|
||||
# router config; MoonEP picks up the [E+B] weight layout for
|
||||
# prefetch and the up projection.
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
|
||||
"""Build the MoE kernel from the layer's current (shuffled) weights."""
|
||||
|
||||
Reference in New Issue
Block a user