[Distributed][MoonEP] BF16 integration of MoonEP balanced EP backend (#52101)

Signed-off-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
Co-authored-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
This commit is contained in:
Kevin Li
2026-09-18 10:34:01 -04:00
committed by GitHub
co-authored by kevinli
parent 2bdae2a5a8
commit 911548247a
15 changed files with 1338 additions and 12 deletions
@@ -0,0 +1,5 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend=moonep"
@@ -10,3 +10,4 @@ Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
Qwen3-30B-A3B-BF16-triton.yaml
Qwen3-30B-A3B-BF16-moonep.yaml
@@ -128,6 +128,12 @@ def test_gsm8k_correctness(config_filename):
"Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. "
"The quantization scheme is not supported on non-GFX950 platforms."
)
if "--all2all-backend=moonep" in eval_config.get("server_args", ""):
from vllm.utils.import_utils import has_moonep
if not has_moonep():
pytest.skip("Skipping MoonEP config: the moonep package is not installed.")
# Parse server arguments from config (use shlex to handle quoted strings)
server_args_str = eval_config.get("server_args", "")
server_args = shlex.split(server_args_str) if server_args_str else []
+412
View File
@@ -0,0 +1,412 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test MoonEP dispatch / prefetch / combine logic (BF16 PoC).
Two paths, both compared against the pure-PyTorch reference MoE:
- MoonEPPrepareAndFinalize + a reference segment-loop expert runner over
MoonEP's expert-grouped ``[NvS, H]`` layout;
- the full modular kernel, MoonEPPrepareAndFinalize + MoonEPExperts through
FusedMoEKernel.
Requires NVSwitch multicast capable GPUs.
"""
import dataclasses
import types
import pytest
import torch
import torch.nn.functional as F
from torch.distributed import ProcessGroup
from tests.kernels.moe.utils import make_test_weights
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.utils.import_utils import has_moonep
from vllm.utils.torch_utils import set_random_seed
from ...utils import multi_gpu_test
from .parallel_utils import ProcessGroupInfo, parallel_launch
if has_moonep():
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
MoonEPExpertWeightLayout,
MoonEPPrepareAndFinalize,
make_moonep_weight_layout,
)
requires_moonep = pytest.mark.skipif(
not has_moonep(),
reason="Requires MoonEP",
)
class MulticastNotAvailableError(RuntimeError):
pass
@dataclasses.dataclass
class TestConfig:
topk: int
m: int
k: int
n: int
num_experts: int
router_skew: float
apply_router_weight_on_input: bool = False
@dataclasses.dataclass
class TestTensors:
rank_tokens: torch.Tensor
topk: torch.Tensor
topk_weights: torch.Tensor
config: TestConfig
@staticmethod
def make(config: TestConfig) -> "TestTensors":
rank_tokens = (
torch.randn((config.m, config.k), device="cuda", dtype=torch.bfloat16) / 10
)
# Skewed routing via a per-expert bias: scaling logits by a scalar
# preserves their ordering and would not change the selected topk
# ids, but a shared bias makes a few experts hot on every token so
# the planner has to fill redundant-expert prefetch slots.
expert_bias = config.router_skew * torch.randn(
1, config.num_experts, device="cuda", dtype=torch.float32
)
logits = (
torch.randn(
config.m, config.num_experts, device="cuda", dtype=torch.float32
)
+ expert_bias
)
topk_weights, topk = torch.topk(logits, config.topk, dim=-1)
topk_weights = torch.softmax(topk_weights, dim=-1)
return TestTensors(
rank_tokens=rank_tokens,
topk=topk.to(dtype=torch.int64),
topk_weights=topk_weights,
config=config,
)
def reference_moonep_experts(
hidden_nvsh: torch.Tensor,
route_weights_nvs: torch.Tensor,
cu_seqlens: torch.Tensor,
weight_layout: "MoonEPExpertWeightLayout",
) -> torch.Tensor:
"""Segment loop over MoonEP's expert-grouped layout.
``prefetch_weight`` has already materialized redundant experts' weights
in rows ``[E, E+B)``, so every segment reads its own row. Route weights
are applied here; MoonEP's combine does the K-sum.
"""
output = torch.empty_like(hidden_nvsh)
prev = 0
for row, cur in enumerate(cu_seqlens.tolist()):
if cur == prev:
continue
x = hidden_nvsh[prev:cur]
gate = F.linear(x, weight_layout.full_gate_weight[row])
up = F.linear(x, weight_layout.full_up_weight[row])
y = F.linear(F.silu(gate) * up, weight_layout.full_down_weight[row])
y = y * route_weights_nvs[prev:cur].to(dtype=y.dtype).unsqueeze(-1)
output[prev:cur].copy_(y)
prev = cur
if prev < hidden_nvsh.shape[0]:
output[prev:].zero_()
return output
def make_moonep_prepare_finalize(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
hidden_size: int,
num_experts: int,
topk: int,
max_tokens_per_rank: int,
weight_layout: "MoonEPExpertWeightLayout",
pass_layout_to_pf: bool = True,
):
from moonep._C import nvl_multicast_supported
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
MoonEPBufferPool,
)
if not nvl_multicast_supported():
raise MulticastNotAvailableError("NVSwitch multicast not available")
pool = MoonEPBufferPool(
dict(
H=hidden_size,
K=topk,
E=num_experts,
num_ep_ranks=pgi.world_size,
B=weight_layout.num_prefetch_slots,
group=pg,
explicitly_destroy=True,
),
max_tokens_per_rank=max_tokens_per_rank,
)
return pool, MoonEPPrepareAndFinalize(
buffer_pool=pool,
max_tokens_per_rank=max_tokens_per_rank,
num_dispatchers=pgi.world_size,
num_global_experts=num_experts,
weight_layout=weight_layout if pass_layout_to_pf else None,
)
def moonep_moe_impl(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
num_prefetch_slots: int,
) -> torch.Tensor:
config = test_tensors.config
hidden_size = test_tensors.rank_tokens.shape[1]
max_tokens_per_rank = 128 * ((config.m + 127) // 128)
weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots)
pool, pf = make_moonep_prepare_finalize(
pg,
pgi,
hidden_size,
config.num_experts,
config.topk,
max_tokens_per_rank,
weight_layout,
)
try:
hidden_nvsh, _, _, _, route_weights_nvs = pf.prepare(
test_tensors.rank_tokens,
test_tensors.topk_weights,
test_tensors.topk,
num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=config.apply_router_weight_on_input,
quant_config=_no_quant_config(),
)
assert route_weights_nvs is not None
if config.apply_router_weight_on_input:
# prepare() already applied the weights to the inputs; the
# runner must not apply them again.
route_weights_nvs = torch.ones_like(route_weights_nvs)
if config.router_skew >= 8 and config.m >= 100:
# Heavy skew must actually engage the redundant-expert planner.
assert (pf.plan.experts_to_copy >= 0).any(), (
"no prefetch slot used despite heavy router skew"
)
expert_out = reference_moonep_experts(
hidden_nvsh, route_weights_nvs, pf.cu_seqlens, weight_layout
)
output = torch.empty_like(test_tensors.rank_tokens)
pf.finalize(
output,
expert_out,
test_tensors.topk_weights,
test_tensors.topk,
apply_router_weight_on_input=False,
weight_and_reduce_impl=TopKWeightAndReduceNoOP(),
)
torch.accelerator.synchronize()
return output
finally:
pool.destroy()
def _no_quant_config():
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
return FusedMoEQuantConfig.make(quant_dtype=None)
def moonep_modular_kernel_impl(
pg: ProcessGroup,
pgi: ProcessGroupInfo,
test_tensors: TestTensors,
w1: torch.Tensor,
w2: torch.Tensor,
num_prefetch_slots: int,
) -> torch.Tensor:
"""Full modular-kernel path: MoonEPPrepareAndFinalize + MoonEPExperts."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
MoonEPExperts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
config = test_tensors.config
hidden_size = test_tensors.rank_tokens.shape[1]
max_tokens_per_rank = 128 * ((config.m + 127) // 128)
weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots)
# weight_layout deliberately not passed to the P/F: the engine path
# resolves it through post_init_setup + the experts' weight hook.
pool, pf = make_moonep_prepare_finalize(
pg,
pgi,
hidden_size,
config.num_experts,
config.topk,
max_tokens_per_rank,
weight_layout,
pass_layout_to_pf=False,
)
try:
moe_config = make_dummy_moe_config(
num_experts=config.num_experts,
experts_per_token=config.topk,
hidden_dim=hidden_size,
intermediate_size=config.n,
max_num_tokens=max_tokens_per_rank,
)
experts = MoonEPExperts(moe_config=moe_config, quant_config=_no_quant_config())
# Stand-in for the layer carrying the layout, as
# convert_to_unquantized_kernel_format leaves it in the engine path.
fake_layer = types.SimpleNamespace(_moonep_weight_layout=weight_layout)
experts.process_weights_after_loading(fake_layer)
kernel = FusedMoEKernel(prepare_finalize=pf, fused_experts=experts)
out = kernel.apply(
hidden_states=test_tensors.rank_tokens,
w1=weight_layout.full_gate_weight,
w2=weight_layout.full_down_weight,
topk_weights=test_tensors.topk_weights,
topk_ids=test_tensors.topk,
activation=MoEActivation.SILU,
global_num_experts=config.num_experts,
expert_map=None,
apply_router_weight_on_input=config.apply_router_weight_on_input,
)
torch.accelerator.synchronize()
return out
finally:
pool.destroy()
def _moonep_moe(
pgi: ProcessGroupInfo,
config: TestConfig,
w1: torch.Tensor,
w2: torch.Tensor,
num_prefetch_slots: int,
use_modular_kernel: bool,
):
from vllm.v1.worker.workspace import init_workspace_manager
device_idx = torch.accelerator.current_device_index()
init_workspace_manager(torch.device("cuda", device_idx))
w1 = w1.to(device=device_idx)
w2 = w2.to(device=device_idx)
pg = torch.distributed.new_group(list(range(pgi.world_size)))
set_random_seed(7 + pgi.rank)
test_tensors = TestTensors.make(config)
with set_current_vllm_config(VllmConfig()):
torch_combined = torch_experts(
test_tensors.rank_tokens,
w1,
w2,
test_tensors.topk_weights,
test_tensors.topk,
apply_router_weights_on_input=config.apply_router_weight_on_input,
)
impl = moonep_modular_kernel_impl if use_modular_kernel else moonep_moe_impl
moonep_combined = impl(pg, pgi, test_tensors, w1, w2, num_prefetch_slots)
torch.testing.assert_close(
torch_combined,
moonep_combined,
atol=6e-2,
rtol=6e-2,
)
MNKs = [
(1, 256, 512),
(37, 256, 512),
(100, 512, 1024),
(512, 768, 2048),
]
@pytest.mark.parametrize("m,n,k", MNKs)
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("topk", [4])
@pytest.mark.parametrize("router_skew", [1.0, 8.0])
@pytest.mark.parametrize("num_prefetch_slots", [4])
@pytest.mark.parametrize("world_size", [2])
@pytest.mark.parametrize("use_modular_kernel", [False, True])
@multi_gpu_test(num_gpus=2)
@requires_moonep
def test_moonep_bf16_moe(
m: int,
n: int,
k: int,
num_experts: int,
topk: int,
router_skew: float,
num_prefetch_slots: int,
world_size: int,
use_modular_kernel: bool,
):
set_random_seed(7)
config = TestConfig(
topk=topk,
m=m,
k=k,
n=n,
num_experts=num_experts,
router_skew=router_skew,
)
(_, w1, _, _), (_, w2, _, _) = make_test_weights(num_experts, n, k)
try:
parallel_launch(
world_size,
_moonep_moe,
config,
w1,
w2,
num_prefetch_slots,
use_modular_kernel,
)
except Exception as exc:
if "MulticastNotAvailableError" in str(exc):
pytest.skip("NVSwitch multicast not available")
raise
@pytest.mark.parametrize("use_modular_kernel", [False, True])
@multi_gpu_test(num_gpus=2)
@requires_moonep
def test_moonep_input_weighted_moe(use_modular_kernel: bool):
"""Llama-4-style routing applies route weights to the expert inputs."""
set_random_seed(7)
config = TestConfig(
topk=1,
m=100,
k=1024,
n=512,
num_experts=32,
router_skew=1.0,
apply_router_weight_on_input=True,
)
(_, w1, _, _), (_, w2, _, _) = make_test_weights(
config.num_experts, config.n, config.k
)
try:
parallel_launch(2, _moonep_moe, config, w1, w2, 4, use_modular_kernel)
except Exception as exc:
if "MulticastNotAvailableError" in str(exc):
pytest.skip("NVSwitch multicast not available")
raise
+2
View File
@@ -49,6 +49,7 @@ All2AllBackend = Literal[
"deepep_v2",
"mori_high_throughput",
"mori_low_latency",
"moonep",
"nixl_ep",
"allgather_reducescatter",
"flashinfer_all2allv", # temporary alias for flashinfer_nvlink_two_sided
@@ -206,6 +207,7 @@ class ParallelConfig:
- "deepep_low_latency": Use deepep low-latency kernels
- "mori_high_throughput": MoRI EP with InterNodeV1 for multi-node
- "mori_low_latency": MoRI EP with InterNodeV1LL for multi-node
- "moonep": MoonEP balanced EP with dynamic redundant experts (NVLink)
- "nixl_ep": Use nixl-ep kernels
- "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels
- "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl"""
+49
View File
@@ -1881,6 +1881,55 @@ class VllmConfig:
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
self._set_compile_ranges()
if self.parallel_config.all2all_backend == "moonep":
if (
self.model_config is not None
and self.model_config.quantization is not None
):
raise ValueError(
"The moonep all2all backend currently supports unquantized "
"BF16 models only; got "
f"quantization={self.model_config.quantization!r}. Use a "
"different --all2all-backend for quantized models."
)
if (
self.model_config is not None
and self.model_config.dtype != torch.bfloat16
):
raise ValueError(
"The moonep all2all backend currently supports BF16 models "
f"only; got dtype={self.model_config.dtype}. Use a "
"different --all2all-backend or --dtype bfloat16."
)
if self.parallel_config.enable_eplb:
raise ValueError(
"The moonep all2all backend does not support EPLB yet: "
"EPLB rearranges expert parameters in a layout MoonEP's "
"replicated [E+B] weights do not follow. Disable "
"--enable-eplb or use a different --all2all-backend."
)
if self.parallel_config.expert_placement_strategy != "linear":
raise ValueError(
"The moonep all2all backend requires linear expert "
"placement: its load-time all-gather assumes each rank "
"holds a contiguous chunk of the global expert range. Got "
"--expert-placement-strategy "
f"{self.parallel_config.expert_placement_strategy!r}."
)
# Enforced here rather than in set_splitting_ops_for_v1 so it
# holds for every compilation mode, and keyed on use_all2all so
# PCP/SP-only topologies are covered too: MoonEP dispatch/combine
# are eager-only and must not be captured.
if (
self.parallel_config.use_all2all
and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
):
logger.info(
"MoonEP: Disabling CUDA Graphs since the MoonEP "
"integration is currently eager-only."
)
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
# Do this after all the updates to compilation_config.mode
effective_dp_size = (
self.parallel_config.data_parallel_size
@@ -21,7 +21,12 @@ from vllm.utils.flashinfer import (
has_flashinfer_nvlink_two_sided,
)
from vllm.utils.func_utils import supports_kw
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2, has_mori
from vllm.utils.import_utils import (
check_moonep_system_support,
has_deep_ep,
has_deep_ep_v2,
has_mori,
)
from .base_device_communicator import All2AllManagerBase, Cache
@@ -1094,3 +1099,71 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
for _, handle in self.handle_cache._cache.items():
handle.destroy()
self.handle_cache._cache.clear()
class MoonEPAll2AllManager(All2AllManagerBase):
"""All2All communication based on MoonEP
(https://github.com/MoonshotAI/MoonEP).
MoonEP keeps token loads perfectly balanced across EP ranks by planning
a small number of dynamically redundant experts online and prefetching
their weights before expert compute. Every rank receives exactly
S x K token slots regardless of router skew, so all communication and
compute shapes are static.
Requires NVLink symmetric-memory / multicast capable topologies
(single node NVSwitch, e.g. H100/H200/B200/GB300 class).
"""
def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
check_moonep_system_support()
super().__init__(cpu_group, tcp_store_group)
self._device_group = device_group
self.handle_cache = Cache()
def _make_buffer_kwargs(
self,
max_num_tokens_per_dp_rank: int,
token_hidden_size: int,
num_topk: int,
num_global_experts: int,
num_prefetch_slots: int,
token_padding: int,
num_sms: int,
) -> dict:
return dict(
S=max_num_tokens_per_dp_rank,
H=token_hidden_size,
K=num_topk,
E=num_global_experts,
num_ep_ranks=self.world_size,
num_sms=num_sms,
token_padding=token_padding,
B=num_prefetch_slots,
group=self._device_group
if self._device_group is not None
else self.cpu_group,
explicitly_destroy=True,
)
def get_handle(self, kwargs):
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
MoonEPBufferPool,
)
buffer_kwargs = self._make_buffer_kwargs(**kwargs)
logger.debug("MoonEP all2all args %s", buffer_kwargs)
def make_pool(**kw):
return MoonEPBufferPool(kw, max_tokens_per_rank=kw["S"])
handle: MoonEPBufferPool = self.handle_cache.get_or_create(
buffer_kwargs, make_pool
)
return handle
def destroy(self):
with self.handle_cache._lock:
for _, handle in self.handle_cache._cache.items():
handle.destroy()
self.handle_cache._cache.clear()
@@ -214,6 +214,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
tcp_store_group,
device_group=self.device_group,
)
elif self.all2all_backend == "moonep":
from .all2all import MoonEPAll2AllManager
self.all2all_manager = MoonEPAll2AllManager(
self.cpu_group,
tcp_store_group,
device_group=self.device_group,
)
elif self.all2all_backend == "nixl_ep":
from .all2all import NixlEPAll2AllManager
@@ -35,6 +35,7 @@ from vllm.platforms import current_platform
from vllm.utils.import_utils import (
has_deep_ep,
has_deep_ep_v2,
has_moonep,
has_mori,
has_nixl_ep,
)
@@ -103,6 +104,13 @@ if current_platform.is_cuda_alike():
NIXL_EP_QUANT_BLOCK_SHAPE,
NixlEPPrepareAndFinalize,
)
if has_moonep():
from .prepare_finalize.moonep import (
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
MOONEP_DEFAULT_NUM_SMS,
MOONEP_DEFAULT_TOKEN_PADDING,
MoonEPPrepareAndFinalize,
)
def get_ep_all2all_manager(
@@ -279,6 +287,28 @@ def maybe_make_prepare_finalize(
sp_size=moe.moe_parallel_config.sp_size,
)
elif moe.use_moonep_kernels:
all_to_all_args = dict(
max_num_tokens_per_dp_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
num_topk=moe.experts_per_token,
num_global_experts=moe.num_experts,
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
token_padding=MOONEP_DEFAULT_TOKEN_PADDING,
num_sms=MOONEP_DEFAULT_NUM_SMS,
)
handle = all2all_manager.get_handle(all_to_all_args)
# The [E+B] weight layout is picked up from the experts (their
# process_weights_after_loading hook) once the layer has loaded and
# converted its weights.
prepare_finalize = MoonEPPrepareAndFinalize(
buffer_pool=handle,
max_tokens_per_rank=moe.max_num_tokens,
num_dispatchers=all2all_manager.world_size,
num_global_experts=moe.num_experts,
)
elif moe.use_mori_kernels:
assert quant_config is not None
@@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig:
def use_deepep_v2_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
@property
def use_moonep_kernels(self):
return self.use_all2all_kernels and self.all2all_backend == "moonep"
@staticmethod
def flatten_tp_across_dp_and_pcp(
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
@@ -1473,6 +1477,10 @@ class FusedMoEConfig:
def use_deepep_v2_kernels(self):
return self.moe_parallel_config.use_deepep_v2_kernels
@property
def use_moonep_kernels(self):
return self.moe_parallel_config.use_moonep_kernels
@property
def needs_round_robin_routing_tables(self):
return self.moe_parallel_config.needs_round_robin_routing_tables
@@ -0,0 +1,220 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations.
MoonEP dispatch delivers tokens already contiguous per weight row
(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute
is three grouped GEMMs over those segments with no permute/unpermute:
gate = grouped_mm(x, w_gate[E+B]) # [NvS, I]
up = grouped_mm(x, w_up[E+B]) # [NvS, I]
act = silu(gate) * up * route_weight # route weights applied here
out = grouped_mm(act, w_down[E+B]) # [NvS, H]
Rows ``[E, E+B)`` are the redundant-expert prefetch slots that
``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments
(including unused prefetch slots) are skipped by the grouped GEMM.
Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be
its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate
tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular
kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the
full layout is picked up from the layer in ``process_weights_after_loading``
and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
from vllm.platforms import current_platform
def moonep_grouped_gemm(
a: torch.Tensor,
w: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> torch.Tensor:
"""``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment.
Args:
a: ``[NvS, K]`` activations in segment order.
w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention).
cu_seqlens: ``[G]`` int32 cumulative segment end offsets.
Returns:
``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled.
"""
return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens)
class MoonEPExperts(mk.FusedMoEExpertsModular):
"""Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16)."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
max_num_tokens: int | None = None,
num_dispatchers: int | None = None,
):
super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers)
assert quant_config.quant_dtype is None, (
"MoonEPExperts supports unquantized BF16 only"
)
if (
quant_config.gemm1_alpha is not None
or quant_config.gemm1_beta is not None
or quant_config.gemm1_clamp_limit is not None
):
raise NotImplementedError(
"MoonEPExperts implements plain silu(gate) * up only; SwiGLU "
"variants with alpha/beta/clamp parameters are not supported"
)
self._weight_layout = None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# The [E+B] gate/up/down layout built by
# convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize
# reads it from here (post_init_setup) for weight prefetch.
layout = getattr(layer, "_moonep_weight_layout", None)
assert layout is not None, (
"MoonEPExperts requires the layer to carry _moonep_weight_layout "
"(set by convert_to_unquantized_kernel_format)"
)
self._weight_layout = layout
@property
def weight_layout(self):
return self._weight_layout
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return current_platform.is_cuda()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation == MoEActivation.SILU
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# EPLB rearranges the named expert parameters as local_num_experts
# rows, which does not understand the replicated [E+B] layout.
return (
moe_parallel_config.use_moonep_kernels
and not moe_parallel_config.enable_eplb
)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key is None and activation_key is None
def supports_chunking(self) -> bool:
# NvS is static and segments must stay whole.
return False
def supports_expert_map(self) -> bool:
# MoonEP addresses global expert rows directly.
return False
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def moe_problem_size(
self,
a1: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
# a1 is [NvS, H] in segment order, not token-major, so the base
# implementation's topk_ids/a1 row-count check does not apply.
assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3
num_rows, intermediate, hidden = w1.shape # [E+B, I, H]
assert a1.size(1) == hidden
return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1)
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
# M is NvS. torch._grouped_mm allocates its own outputs, so no
# workspaces are needed.
return ((0,), (0,), (M, K))
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
# expert_map is ignored: MoonEP dispatches on global expert ids and
# the weights are addressed by global [E+B] row, on every rank.
assert activation == MoEActivation.SILU
assert self._weight_layout is not None, (
"process_weights_after_loading() not called"
)
assert expert_tokens_meta is not None
# MoonEPPrepareAndFinalize returns route weights in NvS order as the
# dispatched topk_weights, and per-row segment lengths as
# expert_tokens_meta.
route_weights_nvs = topk_weights
assert route_weights_nvs.dim() == 1
cu_seqlens = torch.cumsum(
expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32
)
assert cu_seqlens.numel() == w1.size(0)
gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens)
up = moonep_grouped_gemm(
hidden_states, self._weight_layout.full_up_weight, cu_seqlens
)
act = torch.nn.functional.silu(gate)
act.mul_(up)
if not apply_router_weight_on_input:
# Input-weighted routing (top-1, e.g. Llama 4) was already
# applied to the activations by prepare(), before dispatch.
act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1))
# Padding rows past the last segment come out zero-filled.
output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens))
@@ -40,6 +40,7 @@ class UnquantizedMoeBackend(Enum):
AITER = "ROCm AITER"
TRITON = "TRITON"
BATCHED_TRITON = "BATCHED_TRITON"
MOONEP = "MOONEP"
CPU = "CPU"
XPU = "XPU"
TPU = "TPU"
@@ -138,6 +139,13 @@ def backend_to_kernel_cls(
return [BatchedTritonExperts]
elif backend == UnquantizedMoeBackend.MOONEP:
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
MoonEPExperts,
)
return [MoonEPExperts]
elif backend == UnquantizedMoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
@@ -281,6 +289,13 @@ def select_unquantized_moe_backend(
return backend, k_cls
raise ValueError(_make_log_unsupported(backend, reason))
# MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens),
# so it is not interchangeable with the token-major experts backends.
if moe_config.moe_parallel_config.use_moonep_kernels:
return _return_or_raise(
UnquantizedMoeBackend.MOONEP, moe_config, activation_format
)
runner_backend = moe_config.moe_backend
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
# modules_to_not_convert) falls through to auto instead of erroring.
@@ -366,7 +381,42 @@ def convert_to_unquantized_kernel_format(
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
layer: torch.nn.Module | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if unquantized_backend == UnquantizedMoeBackend.MOONEP:
if layer is None:
raise ValueError(
"MoonEP weight conversion requires the layer module to stash "
"the [E+B] weight layout on"
)
# MoonEP addresses expert weights by global [E+B] row and needs one
# contiguous tensor per projection (gate / up / down). The layer's
# w13_weight becomes the gate tensor and w2_weight the down tensor;
# the up tensor and the full layout are stashed on the layer for
# the kernel wiring in _setup_kernel.
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
gather_moonep_weight_layout,
)
if getattr(layer, "_moonep_weight_layout", None) is not None:
# On a reload the layer parameters were already replaced by the
# converted [E+B] gate/down tensors, so the source-format weights
# this conversion needs are gone (and the unregistered up
# projection cannot be recovered from them at all). Tracked in
# RFC #52095.
raise NotImplementedError(
"MoonEP does not support in-place weight reloads yet"
)
layout = gather_moonep_weight_layout(
w13_weight,
w2_weight,
num_global_experts=moe_config.num_experts,
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
)
layer._moonep_weight_layout = layout
return layout.full_gate_weight, layout.full_down_weight
if unquantized_backend == UnquantizedMoeBackend.AITER:
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
w13_weight.is_shuffled = True
@@ -522,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
moe_config: FusedMoEConfig,
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
layer: torch.nn.Module | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
return convert_to_unquantized_kernel_format(
backend, moe_config, w13_weight, w2_weight
backend, moe_config, w13_weight, w2_weight, layer=layer
)
def make_kernel(
@@ -0,0 +1,429 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize.
BF16 correctness-first proof of concept on top of vLLM's modular kernel
interface.
MoonEP differs from DeepEP-style backends in two ways that shape this
integration:
- ``dispatch`` returns tokens already grouped by expert *row* (a fixed
``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together
with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is
no per-token topk id tensor after dispatch; the expert compute must be a
grouped GEMM over ``cu_seqlens`` segments.
- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert
prefetch slots. ``plan.experts_to_copy`` names the source expert of each
slot and ``Buffer.prefetch_weight`` must run between dispatch and expert
compute.
PoC limitations:
- BF16 / unquantized only, eager only.
- Expert weights are replicated in global-expert order on every rank
(memory-heavy). Production Kimi-K3 serving requires sharded
symmetric-memory expert ownership, where rows ``[0, E)`` physically alias
each home rank's parameter memory.
- Route weights are applied inside the expert compute and MoonEP's
``combine`` performs the K-sum, so ``finalize`` requires
``TopKWeightAndReduceNoOP``.
"""
from typing import Any, NamedTuple
import torch
import torch.nn.functional as F
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4
MOONEP_DEFAULT_TOKEN_PADDING = 128
MOONEP_DEFAULT_NUM_SMS = 32
# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128.
MOONEP_WEIGHT_TILE = 128
class MoonEPExpertWeightLayout(NamedTuple):
"""Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout.
Rows ``[0, E)`` hold expert weights in global expert order; rows
``[E, E+B)`` are mutable prefetch slots filled by
``Buffer.prefetch_weight``.
The prefetch slots never need re-zeroing between calls: the planner
marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by
``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so
stale slot contents are never read; used slots are fully overwritten.
"""
full_gate_weight: torch.Tensor
full_up_weight: torch.Tensor
full_down_weight: torch.Tensor
num_prefetch_slots: int
def make_moonep_weight_layout(
w13_weight: torch.Tensor,
w2_weight: torch.Tensor,
num_prefetch_slots: int,
) -> MoonEPExpertWeightLayout:
"""Build the replicated ``[E+B, ...]`` PoC weight layout.
``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight``
``[E, H, I]``, both BF16 in global expert order.
"""
if num_prefetch_slots <= 0:
raise ValueError(
f"num_prefetch_slots must be positive, got {num_prefetch_slots}"
)
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
raise NotImplementedError("MoonEP PoC supports BF16 weights only.")
num_experts, two_i, hidden_size = w13_weight.shape
intermediate_size = two_i // 2
if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size):
raise ValueError(
f"w2_weight shape {tuple(w2_weight.shape)} does not match "
f"w13_weight shape {tuple(w13_weight.shape)}"
)
if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE:
raise ValueError(
"MoonEP weight prefetch requires hidden_size and intermediate_size "
f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, "
f"I={intermediate_size}"
)
full_gate_weight = torch.empty(
num_experts + num_prefetch_slots,
intermediate_size,
hidden_size,
dtype=torch.bfloat16,
device=w13_weight.device,
)
full_up_weight = torch.empty_like(full_gate_weight)
full_down_weight = torch.empty(
num_experts + num_prefetch_slots,
hidden_size,
intermediate_size,
dtype=torch.bfloat16,
device=w2_weight.device,
)
full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :])
full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :])
full_down_weight[:num_experts].copy_(w2_weight)
full_gate_weight[num_experts:].zero_()
full_up_weight[num_experts:].zero_()
full_down_weight[num_experts:].zero_()
return MoonEPExpertWeightLayout(
full_gate_weight=full_gate_weight.contiguous(),
full_up_weight=full_up_weight.contiguous(),
full_down_weight=full_down_weight.contiguous(),
num_prefetch_slots=num_prefetch_slots,
)
def gather_moonep_weight_layout(
w13_local: torch.Tensor,
w2_local: torch.Tensor,
num_global_experts: int,
num_prefetch_slots: int,
) -> MoonEPExpertWeightLayout:
"""Build the replicated ``[E+B, ...]`` layout from this rank's local experts.
PoC bridge: each EP rank loads only its own experts (linear placement),
so all-gather them once at load time into global expert order on every
rank. Production MoonEP instead maps rows ``[0, E)`` onto each home
rank's parameter memory via symmetric memory (RFC #52095 item 5).
"""
from vllm.distributed import get_ep_group
ep_group = get_ep_group()
ep_size = ep_group.world_size
if ep_size == 1:
return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots)
if w13_local.size(0) * ep_size != num_global_experts:
raise NotImplementedError(
"MoonEP PoC requires num_experts to be evenly divisible across EP "
f"ranks: {num_global_experts} experts, {ep_size} ranks"
)
w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0)
w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0)
return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots)
MOONEP_MIN_DISPATCH_TOKENS = 128
class MoonEPBufferPool:
"""Lazily creates ``moonep.Buffer`` instances at power-of-two capacities.
``Buffer`` fixes its token capacity ``S`` at construction, so a single
full-capacity buffer would pad every dispatch to
``max_num_batched_tokens``. The pool instead serves the smallest
power-of-two capacity that fits the step's token count, so decode-sized
batches dispatch a few hundred slots rather than the full capacity.
Selection must be identical on every EP rank (dispatch is collective and
``Buffer`` construction is itself a collective): callers derive the token
count from ``dp_metadata`` (the max across DP ranks) so all ranks create
and pick the same buffer at the same step.
"""
def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int):
self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"}
self.max_tokens_per_rank = max_tokens_per_rank
self._buffers: dict[int, Any] = {}
def capacity_for(self, num_tokens: int) -> int:
if num_tokens >= self.max_tokens_per_rank:
return self.max_tokens_per_rank
cap = 1 << max(num_tokens - 1, 0).bit_length()
return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank)
def get(self, num_tokens: int) -> tuple[int, Any]:
"""Return ``(capacity, buffer)`` for the given step token count."""
from moonep import Buffer # type: ignore[import-not-found]
capacity = self.capacity_for(num_tokens)
buffer = self._buffers.get(capacity)
if buffer is None:
buffer = Buffer(S=capacity, **self._kwargs)
self._buffers[capacity] = buffer
return capacity, buffer
def destroy(self) -> None:
for buffer in self._buffers.values():
buffer.destroy()
self._buffers.clear()
class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
"""Prepare/Finalize using MoonEP balanced dispatch/combine.
``prepare`` pads the batch to the selected buffer's static token
capacity, dispatches, runs ``prefetch_weight`` for the planned redundant
experts, and stashes the ``plan`` for ``finalize`` (the same pattern
DeepEP-HT uses for its handle). Downstream expert compute must consume
the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``.
"""
def __init__(
self,
buffer_pool: "MoonEPBufferPool",
max_tokens_per_rank: int,
num_dispatchers: int,
num_global_experts: int,
weight_layout: MoonEPExpertWeightLayout | None = None,
):
super().__init__()
self.buffer_pool = buffer_pool
self.max_tokens_per_rank = max_tokens_per_rank
self.num_dispatchers_ = num_dispatchers
self.num_global_experts = num_global_experts
self.weight_layout = weight_layout
self._fused_experts: Any = None
# dispatch state consumed by finalize (and the expert runner)
self._plan: Any = None
self._cu_seqlens: torch.Tensor | None = None
self._num_tokens: int = 0
self._buffer: Any = None # the pool buffer used by the current step
def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None:
# The [E+B] weight layout is attached to the experts by their
# process_weights_after_loading hook (after this runs), so keep the
# reference and resolve the layout lazily in prepare().
self._fused_experts = fused_experts
def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout:
if self.weight_layout is None and self._fused_experts is not None:
layout = getattr(self._fused_experts, "weight_layout", None)
if isinstance(layout, MoonEPExpertWeightLayout):
self.weight_layout = layout
# Redundant experts' weights must be in rows [E, E+B) before the
# expert compute reads them; skipping prefetch silently corrupts
# output.
assert self.weight_layout is not None, (
"MoonEPPrepareAndFinalize: weight layout not available (the "
"experts' process_weights_after_loading has not run)"
)
return self.weight_layout
@property
def num_dispatched_slots(self) -> int:
"""``NvS`` of the buffer selected by the current step's prepare()."""
assert self._buffer is not None, "prepare() has not been called"
return int(self._buffer._ctx["NvS"])
def num_dispatchers(self) -> int:
return self.num_dispatchers_
def output_is_reduced(self) -> bool:
# combine returns the fully weighted+reduced token-major output
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
def max_num_tokens_per_rank(self) -> int | None:
return self.max_tokens_per_rank
def topk_indices_dtype(self) -> torch.dtype | None:
return torch.int32
def supports_async(self) -> bool:
return False
@property
def cu_seqlens(self) -> torch.Tensor:
assert self._cu_seqlens is not None, "prepare() has not been called"
return self._cu_seqlens
@property
def plan(self) -> Any:
assert self._plan is not None, "prepare() has not been called"
return self._plan
def _pad_to_capacity(
self,
a1: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
capacity: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
num_tokens = a1.shape[0]
if num_tokens > capacity:
raise ValueError(
f"MoonEP static buffer capacity exceeded: {num_tokens} tokens "
f"> S={capacity}."
)
a1 = a1.contiguous()
topk_ids = topk_ids.to(dtype=torch.int32)
topk_weights = topk_weights.to(dtype=torch.float32)
# MoonEP's planner has no sentinel for invalid expert ids, so map
# negative ids (padded/invalid slots from the engine) to expert 0
# with zero route weight — the same convention as the capacity
# padding rows below. Wasted compute, never wrong output.
invalid = topk_ids < 0
topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids)
topk_weights = torch.where(
invalid, torch.zeros_like(topk_weights), topk_weights
).contiguous()
topk_ids = topk_ids.contiguous()
if num_tokens == capacity:
return a1, topk_ids, topk_weights, num_tokens
pad = capacity - num_tokens
return (
F.pad(a1, (0, 0, 0, pad)),
F.pad(topk_ids, (0, 0, 0, pad)),
F.pad(topk_weights, (0, 0, 0, pad)),
num_tokens,
)
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
if a1.dtype != torch.bfloat16:
raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.")
if quant_config.quant_dtype is not None:
raise NotImplementedError("MoonEP PoC does not support quantized dispatch.")
assert num_experts == self.num_global_experts
assert self._plan is None, (
"MoonEPPrepareAndFinalize.prepare() called again before finalize()"
)
if apply_router_weight_on_input:
assert topk_ids.size(1) == 1, (
"apply_router_weight_on_input requires top-1 routing"
)
a1 = a1 * topk_weights.to(a1.dtype)
# Buffer choice must be rank-symmetric: use the max token count
# across DP ranks (every rank sees the same dp_metadata), falling
# back to the local count outside a forward context (tests).
num_tokens = a1.shape[0]
try:
dp_meta = get_forward_context().dp_metadata
except AssertionError:
dp_meta = None
if dp_meta is not None:
num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max())
capacity, self._buffer = self.buffer_pool.get(num_tokens)
a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity(
a1, topk_ids, topk_weights, capacity
)
tokens_per_expert = torch.bincount(
topk_ids.reshape(-1).to(dtype=torch.int64),
minlength=num_experts,
).to(dtype=torch.int32)
hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch(
a1,
topk_weights,
topk_ids,
tokens_per_expert,
)
self._plan = plan
self._cu_seqlens = cu_seqlens
weight_layout = self._resolve_weight_layout()
self._buffer.prefetch_weight(
plan=plan,
full_gate_weight=weight_layout.full_gate_weight,
full_up_weight=weight_layout.full_up_weight,
full_down_weight=weight_layout.full_down_weight,
)
# Segment sizes per [E+B] weight row. NOTE: this is per *row*, not
# per local expert, and NvS rows are expert-grouped rather than
# token-major — only MoonEP-aware expert implementations can consume
# this activation layout.
expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1))
expert_tokens_meta = mk.ExpertTokensMetadata(
expert_num_tokens=expert_num_tokens,
expert_num_tokens_cpu=None,
)
# MoonEP has no post-dispatch per-token topk ids/weights; route
# weights come back in NvS order.
return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs
def finalize(
self,
output: torch.Tensor,
fused_expert_output: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
apply_router_weight_on_input: bool,
weight_and_reduce_impl: mk.TopKWeightAndReduce,
) -> None:
# Route weights are applied in the expert compute and MoonEP's
# combine performs the K-sum, so any weight/reduce work in finalize
# means MoonEP was paired with the wrong kind of experts.
assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), (
"MoonEP requires TopKWeightAndReduceNoOP, got "
f"{type(weight_and_reduce_impl).__name__}"
)
combined, _, _ = self._buffer.combine(
plan=self.plan,
hidden_nvsh=fused_expert_output,
route_weights_nvs=None,
)
output.copy_(combined[: self._num_tokens])
self._plan = None
self._cu_seqlens = None
@@ -49,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# --8<-- [end:unquantized_fused_moe]
supports_pre_processed_weights = True
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
moe_config=self.moe,
)
# MoonEP replaces the named parameters with its [E+B] gate/down
# split and keeps the up projection outside the parameter registry,
# so a pre-processed (e.g. ipc_cache) load cannot reconstruct it.
self.supports_pre_processed_weights = (
self.unquantized_backend != UnquantizedMoeBackend.MOONEP
)
@property
def supports_eplb(self) -> bool:
@@ -171,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
moe_config=layer.moe_config,
w13_weight=w13,
w2_weight=w2,
layer=layer,
)
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
# On the first call we replace the parameter normally. On subsequent
@@ -191,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
# does not need to be re-built.
self._init_moe_kernel(layer)
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
# The CPU experts need the layer itself for the setup that
# convert_to_unquantized_kernel_format cannot express, since
# it only sees the two weight tensors: padding and prepacking
# into the grouped-gemm layout (bias included), and capturing
# the router config that monolithic apply() cannot carry.
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
# No-op by default. Experts that need the layer itself for setup
# convert_to_unquantized_kernel_format cannot express override
# it: CPU prepacks into its grouped-gemm layout and captures the
# router config; MoonEP picks up the [E+B] weight layout for
# prefetch and the up projection.
assert self.moe_kernel is not None
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
"""Build the MoE kernel from the layer's current (shuffled) weights."""
+28
View File
@@ -512,6 +512,34 @@ def has_nixl_ep() -> bool:
return _has_module("nixl_ep")
def has_moonep() -> bool:
"""Whether the optional `moonep` package is available."""
return _has_module("moonep")
def check_moonep_system_support() -> None:
"""Raise if the current device cannot run MoonEP.
MoonEP's symmetric-memory buffers require CUDA VMM plus NVSwitch multicast
(SHARP) on every EP rank; NVLink-only topologies without NVSwitch (e.g.
4x H100 NV6) fail deep inside ``moonep.Buffer`` otherwise.
"""
if not has_moonep():
raise RuntimeError(
"MoonEP not available. Install it from "
"https://github.com/MoonshotAI/MoonEP."
)
from moonep._C import ( # type: ignore[import-not-found]
nvl_multicast_supported,
)
if not nvl_multicast_supported():
raise RuntimeError(
"MoonEP requires NVSwitch multicast (SHARP) support on the "
"current device; it is not supported on this GPU/topology."
)
def is_numba_available() -> bool:
"""Whether the optional `numba` package is available."""
return _has_module("numba")