[Distributed][MoonEP] BF16 integration of MoonEP balanced EP backend (#52101)
Signed-off-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com> Co-authored-by: kevinli <306243285+kaijunli-infr@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
model_name: "Qwen/Qwen3-30B-A3B"
|
||||
accuracy_threshold: 0.88
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend=moonep"
|
||||
@@ -10,3 +10,4 @@ Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
|
||||
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
|
||||
Qwen3-30B-A3B-BF16-triton.yaml
|
||||
Qwen3-30B-A3B-BF16-moonep.yaml
|
||||
|
||||
@@ -128,6 +128,12 @@ def test_gsm8k_correctness(config_filename):
|
||||
"Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. "
|
||||
"The quantization scheme is not supported on non-GFX950 platforms."
|
||||
)
|
||||
if "--all2all-backend=moonep" in eval_config.get("server_args", ""):
|
||||
from vllm.utils.import_utils import has_moonep
|
||||
|
||||
if not has_moonep():
|
||||
pytest.skip("Skipping MoonEP config: the moonep package is not installed.")
|
||||
|
||||
# Parse server arguments from config (use shlex to handle quoted strings)
|
||||
server_args_str = eval_config.get("server_args", "")
|
||||
server_args = shlex.split(server_args_str) if server_args_str else []
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test MoonEP dispatch / prefetch / combine logic (BF16 PoC).
|
||||
|
||||
Two paths, both compared against the pure-PyTorch reference MoE:
|
||||
- MoonEPPrepareAndFinalize + a reference segment-loop expert runner over
|
||||
MoonEP's expert-grouped ``[NvS, H]`` layout;
|
||||
- the full modular kernel, MoonEPPrepareAndFinalize + MoonEPExperts through
|
||||
FusedMoEKernel.
|
||||
Requires NVSwitch multicast capable GPUs.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from tests.kernels.moe.utils import make_test_weights
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.utils.import_utils import has_moonep
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .parallel_utils import ProcessGroupInfo, parallel_launch
|
||||
|
||||
if has_moonep():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MoonEPExpertWeightLayout,
|
||||
MoonEPPrepareAndFinalize,
|
||||
make_moonep_weight_layout,
|
||||
)
|
||||
|
||||
requires_moonep = pytest.mark.skipif(
|
||||
not has_moonep(),
|
||||
reason="Requires MoonEP",
|
||||
)
|
||||
|
||||
|
||||
class MulticastNotAvailableError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestConfig:
|
||||
topk: int
|
||||
m: int
|
||||
k: int
|
||||
n: int
|
||||
num_experts: int
|
||||
router_skew: float
|
||||
apply_router_weight_on_input: bool = False
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestTensors:
|
||||
rank_tokens: torch.Tensor
|
||||
topk: torch.Tensor
|
||||
topk_weights: torch.Tensor
|
||||
config: TestConfig
|
||||
|
||||
@staticmethod
|
||||
def make(config: TestConfig) -> "TestTensors":
|
||||
rank_tokens = (
|
||||
torch.randn((config.m, config.k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
# Skewed routing via a per-expert bias: scaling logits by a scalar
|
||||
# preserves their ordering and would not change the selected topk
|
||||
# ids, but a shared bias makes a few experts hot on every token so
|
||||
# the planner has to fill redundant-expert prefetch slots.
|
||||
expert_bias = config.router_skew * torch.randn(
|
||||
1, config.num_experts, device="cuda", dtype=torch.float32
|
||||
)
|
||||
logits = (
|
||||
torch.randn(
|
||||
config.m, config.num_experts, device="cuda", dtype=torch.float32
|
||||
)
|
||||
+ expert_bias
|
||||
)
|
||||
topk_weights, topk = torch.topk(logits, config.topk, dim=-1)
|
||||
topk_weights = torch.softmax(topk_weights, dim=-1)
|
||||
return TestTensors(
|
||||
rank_tokens=rank_tokens,
|
||||
topk=topk.to(dtype=torch.int64),
|
||||
topk_weights=topk_weights,
|
||||
config=config,
|
||||
)
|
||||
|
||||
|
||||
def reference_moonep_experts(
|
||||
hidden_nvsh: torch.Tensor,
|
||||
route_weights_nvs: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
weight_layout: "MoonEPExpertWeightLayout",
|
||||
) -> torch.Tensor:
|
||||
"""Segment loop over MoonEP's expert-grouped layout.
|
||||
|
||||
``prefetch_weight`` has already materialized redundant experts' weights
|
||||
in rows ``[E, E+B)``, so every segment reads its own row. Route weights
|
||||
are applied here; MoonEP's combine does the K-sum.
|
||||
"""
|
||||
output = torch.empty_like(hidden_nvsh)
|
||||
prev = 0
|
||||
for row, cur in enumerate(cu_seqlens.tolist()):
|
||||
if cur == prev:
|
||||
continue
|
||||
x = hidden_nvsh[prev:cur]
|
||||
gate = F.linear(x, weight_layout.full_gate_weight[row])
|
||||
up = F.linear(x, weight_layout.full_up_weight[row])
|
||||
y = F.linear(F.silu(gate) * up, weight_layout.full_down_weight[row])
|
||||
y = y * route_weights_nvs[prev:cur].to(dtype=y.dtype).unsqueeze(-1)
|
||||
output[prev:cur].copy_(y)
|
||||
prev = cur
|
||||
if prev < hidden_nvsh.shape[0]:
|
||||
output[prev:].zero_()
|
||||
return output
|
||||
|
||||
|
||||
def make_moonep_prepare_finalize(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
max_tokens_per_rank: int,
|
||||
weight_layout: "MoonEPExpertWeightLayout",
|
||||
pass_layout_to_pf: bool = True,
|
||||
):
|
||||
from moonep._C import nvl_multicast_supported
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MoonEPBufferPool,
|
||||
)
|
||||
|
||||
if not nvl_multicast_supported():
|
||||
raise MulticastNotAvailableError("NVSwitch multicast not available")
|
||||
|
||||
pool = MoonEPBufferPool(
|
||||
dict(
|
||||
H=hidden_size,
|
||||
K=topk,
|
||||
E=num_experts,
|
||||
num_ep_ranks=pgi.world_size,
|
||||
B=weight_layout.num_prefetch_slots,
|
||||
group=pg,
|
||||
explicitly_destroy=True,
|
||||
),
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
)
|
||||
return pool, MoonEPPrepareAndFinalize(
|
||||
buffer_pool=pool,
|
||||
max_tokens_per_rank=max_tokens_per_rank,
|
||||
num_dispatchers=pgi.world_size,
|
||||
num_global_experts=num_experts,
|
||||
weight_layout=weight_layout if pass_layout_to_pf else None,
|
||||
)
|
||||
|
||||
|
||||
def moonep_moe_impl(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
test_tensors: TestTensors,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_prefetch_slots: int,
|
||||
) -> torch.Tensor:
|
||||
config = test_tensors.config
|
||||
hidden_size = test_tensors.rank_tokens.shape[1]
|
||||
max_tokens_per_rank = 128 * ((config.m + 127) // 128)
|
||||
|
||||
weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots)
|
||||
pool, pf = make_moonep_prepare_finalize(
|
||||
pg,
|
||||
pgi,
|
||||
hidden_size,
|
||||
config.num_experts,
|
||||
config.topk,
|
||||
max_tokens_per_rank,
|
||||
weight_layout,
|
||||
)
|
||||
try:
|
||||
hidden_nvsh, _, _, _, route_weights_nvs = pf.prepare(
|
||||
test_tensors.rank_tokens,
|
||||
test_tensors.topk_weights,
|
||||
test_tensors.topk,
|
||||
num_experts=config.num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=config.apply_router_weight_on_input,
|
||||
quant_config=_no_quant_config(),
|
||||
)
|
||||
assert route_weights_nvs is not None
|
||||
if config.apply_router_weight_on_input:
|
||||
# prepare() already applied the weights to the inputs; the
|
||||
# runner must not apply them again.
|
||||
route_weights_nvs = torch.ones_like(route_weights_nvs)
|
||||
if config.router_skew >= 8 and config.m >= 100:
|
||||
# Heavy skew must actually engage the redundant-expert planner.
|
||||
assert (pf.plan.experts_to_copy >= 0).any(), (
|
||||
"no prefetch slot used despite heavy router skew"
|
||||
)
|
||||
expert_out = reference_moonep_experts(
|
||||
hidden_nvsh, route_weights_nvs, pf.cu_seqlens, weight_layout
|
||||
)
|
||||
output = torch.empty_like(test_tensors.rank_tokens)
|
||||
pf.finalize(
|
||||
output,
|
||||
expert_out,
|
||||
test_tensors.topk_weights,
|
||||
test_tensors.topk,
|
||||
apply_router_weight_on_input=False,
|
||||
weight_and_reduce_impl=TopKWeightAndReduceNoOP(),
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return output
|
||||
finally:
|
||||
pool.destroy()
|
||||
|
||||
|
||||
def _no_quant_config():
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
|
||||
return FusedMoEQuantConfig.make(quant_dtype=None)
|
||||
|
||||
|
||||
def moonep_modular_kernel_impl(
|
||||
pg: ProcessGroup,
|
||||
pgi: ProcessGroupInfo,
|
||||
test_tensors: TestTensors,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_prefetch_slots: int,
|
||||
) -> torch.Tensor:
|
||||
"""Full modular-kernel path: MoonEPPrepareAndFinalize + MoonEPExperts."""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
|
||||
MoonEPExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
|
||||
config = test_tensors.config
|
||||
hidden_size = test_tensors.rank_tokens.shape[1]
|
||||
max_tokens_per_rank = 128 * ((config.m + 127) // 128)
|
||||
|
||||
weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots)
|
||||
# weight_layout deliberately not passed to the P/F: the engine path
|
||||
# resolves it through post_init_setup + the experts' weight hook.
|
||||
pool, pf = make_moonep_prepare_finalize(
|
||||
pg,
|
||||
pgi,
|
||||
hidden_size,
|
||||
config.num_experts,
|
||||
config.topk,
|
||||
max_tokens_per_rank,
|
||||
weight_layout,
|
||||
pass_layout_to_pf=False,
|
||||
)
|
||||
try:
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=config.num_experts,
|
||||
experts_per_token=config.topk,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=config.n,
|
||||
max_num_tokens=max_tokens_per_rank,
|
||||
)
|
||||
experts = MoonEPExperts(moe_config=moe_config, quant_config=_no_quant_config())
|
||||
# Stand-in for the layer carrying the layout, as
|
||||
# convert_to_unquantized_kernel_format leaves it in the engine path.
|
||||
fake_layer = types.SimpleNamespace(_moonep_weight_layout=weight_layout)
|
||||
experts.process_weights_after_loading(fake_layer)
|
||||
kernel = FusedMoEKernel(prepare_finalize=pf, fused_experts=experts)
|
||||
out = kernel.apply(
|
||||
hidden_states=test_tensors.rank_tokens,
|
||||
w1=weight_layout.full_gate_weight,
|
||||
w2=weight_layout.full_down_weight,
|
||||
topk_weights=test_tensors.topk_weights,
|
||||
topk_ids=test_tensors.topk,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=config.num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=config.apply_router_weight_on_input,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return out
|
||||
finally:
|
||||
pool.destroy()
|
||||
|
||||
|
||||
def _moonep_moe(
|
||||
pgi: ProcessGroupInfo,
|
||||
config: TestConfig,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
num_prefetch_slots: int,
|
||||
use_modular_kernel: bool,
|
||||
):
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
device_idx = torch.accelerator.current_device_index()
|
||||
init_workspace_manager(torch.device("cuda", device_idx))
|
||||
w1 = w1.to(device=device_idx)
|
||||
w2 = w2.to(device=device_idx)
|
||||
|
||||
pg = torch.distributed.new_group(list(range(pgi.world_size)))
|
||||
set_random_seed(7 + pgi.rank)
|
||||
test_tensors = TestTensors.make(config)
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
torch_combined = torch_experts(
|
||||
test_tensors.rank_tokens,
|
||||
w1,
|
||||
w2,
|
||||
test_tensors.topk_weights,
|
||||
test_tensors.topk,
|
||||
apply_router_weights_on_input=config.apply_router_weight_on_input,
|
||||
)
|
||||
impl = moonep_modular_kernel_impl if use_modular_kernel else moonep_moe_impl
|
||||
moonep_combined = impl(pg, pgi, test_tensors, w1, w2, num_prefetch_slots)
|
||||
|
||||
torch.testing.assert_close(
|
||||
torch_combined,
|
||||
moonep_combined,
|
||||
atol=6e-2,
|
||||
rtol=6e-2,
|
||||
)
|
||||
|
||||
|
||||
MNKs = [
|
||||
(1, 256, 512),
|
||||
(37, 256, 512),
|
||||
(100, 512, 1024),
|
||||
(512, 768, 2048),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNKs)
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("topk", [4])
|
||||
@pytest.mark.parametrize("router_skew", [1.0, 8.0])
|
||||
@pytest.mark.parametrize("num_prefetch_slots", [4])
|
||||
@pytest.mark.parametrize("world_size", [2])
|
||||
@pytest.mark.parametrize("use_modular_kernel", [False, True])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_moonep
|
||||
def test_moonep_bf16_moe(
|
||||
m: int,
|
||||
n: int,
|
||||
k: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
router_skew: float,
|
||||
num_prefetch_slots: int,
|
||||
world_size: int,
|
||||
use_modular_kernel: bool,
|
||||
):
|
||||
set_random_seed(7)
|
||||
config = TestConfig(
|
||||
topk=topk,
|
||||
m=m,
|
||||
k=k,
|
||||
n=n,
|
||||
num_experts=num_experts,
|
||||
router_skew=router_skew,
|
||||
)
|
||||
(_, w1, _, _), (_, w2, _, _) = make_test_weights(num_experts, n, k)
|
||||
|
||||
try:
|
||||
parallel_launch(
|
||||
world_size,
|
||||
_moonep_moe,
|
||||
config,
|
||||
w1,
|
||||
w2,
|
||||
num_prefetch_slots,
|
||||
use_modular_kernel,
|
||||
)
|
||||
except Exception as exc:
|
||||
if "MulticastNotAvailableError" in str(exc):
|
||||
pytest.skip("NVSwitch multicast not available")
|
||||
raise
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_modular_kernel", [False, True])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@requires_moonep
|
||||
def test_moonep_input_weighted_moe(use_modular_kernel: bool):
|
||||
"""Llama-4-style routing applies route weights to the expert inputs."""
|
||||
set_random_seed(7)
|
||||
config = TestConfig(
|
||||
topk=1,
|
||||
m=100,
|
||||
k=1024,
|
||||
n=512,
|
||||
num_experts=32,
|
||||
router_skew=1.0,
|
||||
apply_router_weight_on_input=True,
|
||||
)
|
||||
(_, w1, _, _), (_, w2, _, _) = make_test_weights(
|
||||
config.num_experts, config.n, config.k
|
||||
)
|
||||
try:
|
||||
parallel_launch(2, _moonep_moe, config, w1, w2, 4, use_modular_kernel)
|
||||
except Exception as exc:
|
||||
if "MulticastNotAvailableError" in str(exc):
|
||||
pytest.skip("NVSwitch multicast not available")
|
||||
raise
|
||||
@@ -49,6 +49,7 @@ All2AllBackend = Literal[
|
||||
"deepep_v2",
|
||||
"mori_high_throughput",
|
||||
"mori_low_latency",
|
||||
"moonep",
|
||||
"nixl_ep",
|
||||
"allgather_reducescatter",
|
||||
"flashinfer_all2allv", # temporary alias for flashinfer_nvlink_two_sided
|
||||
@@ -206,6 +207,7 @@ class ParallelConfig:
|
||||
- "deepep_low_latency": Use deepep low-latency kernels
|
||||
- "mori_high_throughput": MoRI EP with InterNodeV1 for multi-node
|
||||
- "mori_low_latency": MoRI EP with InterNodeV1LL for multi-node
|
||||
- "moonep": MoonEP balanced EP with dynamic redundant experts (NVLink)
|
||||
- "nixl_ep": Use nixl-ep kernels
|
||||
- "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels
|
||||
- "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl"""
|
||||
|
||||
@@ -1881,6 +1881,55 @@ class VllmConfig:
|
||||
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
|
||||
self._set_compile_ranges()
|
||||
|
||||
if self.parallel_config.all2all_backend == "moonep":
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.quantization is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"The moonep all2all backend currently supports unquantized "
|
||||
"BF16 models only; got "
|
||||
f"quantization={self.model_config.quantization!r}. Use a "
|
||||
"different --all2all-backend for quantized models."
|
||||
)
|
||||
if (
|
||||
self.model_config is not None
|
||||
and self.model_config.dtype != torch.bfloat16
|
||||
):
|
||||
raise ValueError(
|
||||
"The moonep all2all backend currently supports BF16 models "
|
||||
f"only; got dtype={self.model_config.dtype}. Use a "
|
||||
"different --all2all-backend or --dtype bfloat16."
|
||||
)
|
||||
if self.parallel_config.enable_eplb:
|
||||
raise ValueError(
|
||||
"The moonep all2all backend does not support EPLB yet: "
|
||||
"EPLB rearranges expert parameters in a layout MoonEP's "
|
||||
"replicated [E+B] weights do not follow. Disable "
|
||||
"--enable-eplb or use a different --all2all-backend."
|
||||
)
|
||||
if self.parallel_config.expert_placement_strategy != "linear":
|
||||
raise ValueError(
|
||||
"The moonep all2all backend requires linear expert "
|
||||
"placement: its load-time all-gather assumes each rank "
|
||||
"holds a contiguous chunk of the global expert range. Got "
|
||||
"--expert-placement-strategy "
|
||||
f"{self.parallel_config.expert_placement_strategy!r}."
|
||||
)
|
||||
# Enforced here rather than in set_splitting_ops_for_v1 so it
|
||||
# holds for every compilation mode, and keyed on use_all2all so
|
||||
# PCP/SP-only topologies are covered too: MoonEP dispatch/combine
|
||||
# are eager-only and must not be captured.
|
||||
if (
|
||||
self.parallel_config.use_all2all
|
||||
and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
|
||||
):
|
||||
logger.info(
|
||||
"MoonEP: Disabling CUDA Graphs since the MoonEP "
|
||||
"integration is currently eager-only."
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# Do this after all the updates to compilation_config.mode
|
||||
effective_dp_size = (
|
||||
self.parallel_config.data_parallel_size
|
||||
|
||||
@@ -21,7 +21,12 @@ from vllm.utils.flashinfer import (
|
||||
has_flashinfer_nvlink_two_sided,
|
||||
)
|
||||
from vllm.utils.func_utils import supports_kw
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2, has_mori
|
||||
from vllm.utils.import_utils import (
|
||||
check_moonep_system_support,
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_mori,
|
||||
)
|
||||
|
||||
from .base_device_communicator import All2AllManagerBase, Cache
|
||||
|
||||
@@ -1094,3 +1099,71 @@ class DeepEPV2All2AllManager(All2AllManagerBase):
|
||||
for _, handle in self.handle_cache._cache.items():
|
||||
handle.destroy()
|
||||
self.handle_cache._cache.clear()
|
||||
|
||||
|
||||
class MoonEPAll2AllManager(All2AllManagerBase):
|
||||
"""All2All communication based on MoonEP
|
||||
(https://github.com/MoonshotAI/MoonEP).
|
||||
|
||||
MoonEP keeps token loads perfectly balanced across EP ranks by planning
|
||||
a small number of dynamically redundant experts online and prefetching
|
||||
their weights before expert compute. Every rank receives exactly
|
||||
S x K token slots regardless of router skew, so all communication and
|
||||
compute shapes are static.
|
||||
|
||||
Requires NVLink symmetric-memory / multicast capable topologies
|
||||
(single node NVSwitch, e.g. H100/H200/B200/GB300 class).
|
||||
"""
|
||||
|
||||
def __init__(self, cpu_group, tcp_store_group=None, device_group=None):
|
||||
check_moonep_system_support()
|
||||
super().__init__(cpu_group, tcp_store_group)
|
||||
self._device_group = device_group
|
||||
self.handle_cache = Cache()
|
||||
|
||||
def _make_buffer_kwargs(
|
||||
self,
|
||||
max_num_tokens_per_dp_rank: int,
|
||||
token_hidden_size: int,
|
||||
num_topk: int,
|
||||
num_global_experts: int,
|
||||
num_prefetch_slots: int,
|
||||
token_padding: int,
|
||||
num_sms: int,
|
||||
) -> dict:
|
||||
return dict(
|
||||
S=max_num_tokens_per_dp_rank,
|
||||
H=token_hidden_size,
|
||||
K=num_topk,
|
||||
E=num_global_experts,
|
||||
num_ep_ranks=self.world_size,
|
||||
num_sms=num_sms,
|
||||
token_padding=token_padding,
|
||||
B=num_prefetch_slots,
|
||||
group=self._device_group
|
||||
if self._device_group is not None
|
||||
else self.cpu_group,
|
||||
explicitly_destroy=True,
|
||||
)
|
||||
|
||||
def get_handle(self, kwargs):
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MoonEPBufferPool,
|
||||
)
|
||||
|
||||
buffer_kwargs = self._make_buffer_kwargs(**kwargs)
|
||||
logger.debug("MoonEP all2all args %s", buffer_kwargs)
|
||||
|
||||
def make_pool(**kw):
|
||||
return MoonEPBufferPool(kw, max_tokens_per_rank=kw["S"])
|
||||
|
||||
handle: MoonEPBufferPool = self.handle_cache.get_or_create(
|
||||
buffer_kwargs, make_pool
|
||||
)
|
||||
return handle
|
||||
|
||||
def destroy(self):
|
||||
with self.handle_cache._lock:
|
||||
for _, handle in self.handle_cache._cache.items():
|
||||
handle.destroy()
|
||||
self.handle_cache._cache.clear()
|
||||
|
||||
@@ -214,6 +214,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
tcp_store_group,
|
||||
device_group=self.device_group,
|
||||
)
|
||||
elif self.all2all_backend == "moonep":
|
||||
from .all2all import MoonEPAll2AllManager
|
||||
|
||||
self.all2all_manager = MoonEPAll2AllManager(
|
||||
self.cpu_group,
|
||||
tcp_store_group,
|
||||
device_group=self.device_group,
|
||||
)
|
||||
elif self.all2all_backend == "nixl_ep":
|
||||
from .all2all import NixlEPAll2AllManager
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import (
|
||||
has_deep_ep,
|
||||
has_deep_ep_v2,
|
||||
has_moonep,
|
||||
has_mori,
|
||||
has_nixl_ep,
|
||||
)
|
||||
@@ -103,6 +104,13 @@ if current_platform.is_cuda_alike():
|
||||
NIXL_EP_QUANT_BLOCK_SHAPE,
|
||||
NixlEPPrepareAndFinalize,
|
||||
)
|
||||
if has_moonep():
|
||||
from .prepare_finalize.moonep import (
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
MOONEP_DEFAULT_NUM_SMS,
|
||||
MOONEP_DEFAULT_TOKEN_PADDING,
|
||||
MoonEPPrepareAndFinalize,
|
||||
)
|
||||
|
||||
|
||||
def get_ep_all2all_manager(
|
||||
@@ -279,6 +287,28 @@ def maybe_make_prepare_finalize(
|
||||
sp_size=moe.moe_parallel_config.sp_size,
|
||||
)
|
||||
|
||||
elif moe.use_moonep_kernels:
|
||||
all_to_all_args = dict(
|
||||
max_num_tokens_per_dp_rank=moe.max_num_tokens,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
num_topk=moe.experts_per_token,
|
||||
num_global_experts=moe.num_experts,
|
||||
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
token_padding=MOONEP_DEFAULT_TOKEN_PADDING,
|
||||
num_sms=MOONEP_DEFAULT_NUM_SMS,
|
||||
)
|
||||
handle = all2all_manager.get_handle(all_to_all_args)
|
||||
|
||||
# The [E+B] weight layout is picked up from the experts (their
|
||||
# process_weights_after_loading hook) once the layer has loaded and
|
||||
# converted its weights.
|
||||
prepare_finalize = MoonEPPrepareAndFinalize(
|
||||
buffer_pool=handle,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
num_dispatchers=all2all_manager.world_size,
|
||||
num_global_experts=moe.num_experts,
|
||||
)
|
||||
|
||||
elif moe.use_mori_kernels:
|
||||
assert quant_config is not None
|
||||
|
||||
|
||||
@@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig:
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "deepep_v2"
|
||||
|
||||
@property
|
||||
def use_moonep_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend == "moonep"
|
||||
|
||||
@staticmethod
|
||||
def flatten_tp_across_dp_and_pcp(
|
||||
tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int
|
||||
@@ -1473,6 +1477,10 @@ class FusedMoEConfig:
|
||||
def use_deepep_v2_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_v2_kernels
|
||||
|
||||
@property
|
||||
def use_moonep_kernels(self):
|
||||
return self.moe_parallel_config.use_moonep_kernels
|
||||
|
||||
@property
|
||||
def needs_round_robin_routing_tables(self):
|
||||
return self.moe_parallel_config.needs_round_robin_routing_tables
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations.
|
||||
|
||||
MoonEP dispatch delivers tokens already contiguous per weight row
|
||||
(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute
|
||||
is three grouped GEMMs over those segments with no permute/unpermute:
|
||||
|
||||
gate = grouped_mm(x, w_gate[E+B]) # [NvS, I]
|
||||
up = grouped_mm(x, w_up[E+B]) # [NvS, I]
|
||||
act = silu(gate) * up * route_weight # route weights applied here
|
||||
out = grouped_mm(act, w_down[E+B]) # [NvS, H]
|
||||
|
||||
Rows ``[E, E+B)`` are the redundant-expert prefetch slots that
|
||||
``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments
|
||||
(including unused prefetch slots) are skipped by the grouped GEMM.
|
||||
|
||||
Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be
|
||||
its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate
|
||||
tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular
|
||||
kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the
|
||||
full layout is picked up from the layer in ``process_weights_after_loading``
|
||||
and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def moonep_grouped_gemm(
|
||||
a: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment.
|
||||
|
||||
Args:
|
||||
a: ``[NvS, K]`` activations in segment order.
|
||||
w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention).
|
||||
cu_seqlens: ``[G]`` int32 cumulative segment end offsets.
|
||||
|
||||
Returns:
|
||||
``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled.
|
||||
|
||||
"""
|
||||
return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens)
|
||||
|
||||
|
||||
class MoonEPExperts(mk.FusedMoEExpertsModular):
|
||||
"""Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
max_num_tokens: int | None = None,
|
||||
num_dispatchers: int | None = None,
|
||||
):
|
||||
super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers)
|
||||
assert quant_config.quant_dtype is None, (
|
||||
"MoonEPExperts supports unquantized BF16 only"
|
||||
)
|
||||
if (
|
||||
quant_config.gemm1_alpha is not None
|
||||
or quant_config.gemm1_beta is not None
|
||||
or quant_config.gemm1_clamp_limit is not None
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MoonEPExperts implements plain silu(gate) * up only; SwiGLU "
|
||||
"variants with alpha/beta/clamp parameters are not supported"
|
||||
)
|
||||
self._weight_layout = None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# The [E+B] gate/up/down layout built by
|
||||
# convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize
|
||||
# reads it from here (post_init_setup) for weight prefetch.
|
||||
layout = getattr(layer, "_moonep_weight_layout", None)
|
||||
assert layout is not None, (
|
||||
"MoonEPExperts requires the layer to carry _moonep_weight_layout "
|
||||
"(set by convert_to_unquantized_kernel_format)"
|
||||
)
|
||||
self._weight_layout = layout
|
||||
|
||||
@property
|
||||
def weight_layout(self):
|
||||
return self._weight_layout
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return current_platform.is_cuda()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation == MoEActivation.SILU
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# EPLB rearranges the named expert parameters as local_num_experts
|
||||
# rows, which does not understand the replicated [E+B] layout.
|
||||
return (
|
||||
moe_parallel_config.use_moonep_kernels
|
||||
and not moe_parallel_config.enable_eplb
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key is None and activation_key is None
|
||||
|
||||
def supports_chunking(self) -> bool:
|
||||
# NvS is static and segments must stay whole.
|
||||
return False
|
||||
|
||||
def supports_expert_map(self) -> bool:
|
||||
# MoonEP addresses global expert rows directly.
|
||||
return False
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
# a1 is [NvS, H] in segment order, not token-major, so the base
|
||||
# implementation's topk_ids/a1 row-count check does not apply.
|
||||
assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3
|
||||
num_rows, intermediate, hidden = w1.shape # [E+B, I, H]
|
||||
assert a1.size(1) == hidden
|
||||
return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
# M is NvS. torch._grouped_mm allocates its own outputs, so no
|
||||
# workspaces are needed.
|
||||
return ((0,), (0,), (M, K))
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
# expert_map is ignored: MoonEP dispatches on global expert ids and
|
||||
# the weights are addressed by global [E+B] row, on every rank.
|
||||
assert activation == MoEActivation.SILU
|
||||
assert self._weight_layout is not None, (
|
||||
"process_weights_after_loading() not called"
|
||||
)
|
||||
assert expert_tokens_meta is not None
|
||||
# MoonEPPrepareAndFinalize returns route weights in NvS order as the
|
||||
# dispatched topk_weights, and per-row segment lengths as
|
||||
# expert_tokens_meta.
|
||||
route_weights_nvs = topk_weights
|
||||
assert route_weights_nvs.dim() == 1
|
||||
cu_seqlens = torch.cumsum(
|
||||
expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32
|
||||
)
|
||||
|
||||
assert cu_seqlens.numel() == w1.size(0)
|
||||
|
||||
gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens)
|
||||
up = moonep_grouped_gemm(
|
||||
hidden_states, self._weight_layout.full_up_weight, cu_seqlens
|
||||
)
|
||||
act = torch.nn.functional.silu(gate)
|
||||
act.mul_(up)
|
||||
if not apply_router_weight_on_input:
|
||||
# Input-weighted routing (top-1, e.g. Llama 4) was already
|
||||
# applied to the activations by prepare(), before dispatch.
|
||||
act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1))
|
||||
# Padding rows past the last segment come out zero-filled.
|
||||
output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens))
|
||||
@@ -40,6 +40,7 @@ class UnquantizedMoeBackend(Enum):
|
||||
AITER = "ROCm AITER"
|
||||
TRITON = "TRITON"
|
||||
BATCHED_TRITON = "BATCHED_TRITON"
|
||||
MOONEP = "MOONEP"
|
||||
CPU = "CPU"
|
||||
XPU = "XPU"
|
||||
TPU = "TPU"
|
||||
@@ -138,6 +139,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [BatchedTritonExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.MOONEP:
|
||||
from vllm.model_executor.layers.fused_moe.experts.moonep_experts import (
|
||||
MoonEPExperts,
|
||||
)
|
||||
|
||||
return [MoonEPExperts]
|
||||
|
||||
elif backend == UnquantizedMoeBackend.XPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts
|
||||
|
||||
@@ -281,6 +289,13 @@ def select_unquantized_moe_backend(
|
||||
return backend, k_cls
|
||||
raise ValueError(_make_log_unsupported(backend, reason))
|
||||
|
||||
# MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens),
|
||||
# so it is not interchangeable with the token-major experts backends.
|
||||
if moe_config.moe_parallel_config.use_moonep_kernels:
|
||||
return _return_or_raise(
|
||||
UnquantizedMoeBackend.MOONEP, moe_config, activation_format
|
||||
)
|
||||
|
||||
runner_backend = moe_config.moe_backend
|
||||
# 'humming' is quantization-only; an unquantized layer (e.g. excluded via
|
||||
# modules_to_not_convert) falls through to auto instead of erroring.
|
||||
@@ -366,7 +381,42 @@ def convert_to_unquantized_kernel_format(
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if unquantized_backend == UnquantizedMoeBackend.MOONEP:
|
||||
if layer is None:
|
||||
raise ValueError(
|
||||
"MoonEP weight conversion requires the layer module to stash "
|
||||
"the [E+B] weight layout on"
|
||||
)
|
||||
# MoonEP addresses expert weights by global [E+B] row and needs one
|
||||
# contiguous tensor per projection (gate / up / down). The layer's
|
||||
# w13_weight becomes the gate tensor and w2_weight the down tensor;
|
||||
# the up tensor and the full layout are stashed on the layer for
|
||||
# the kernel wiring in _setup_kernel.
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import (
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
gather_moonep_weight_layout,
|
||||
)
|
||||
|
||||
if getattr(layer, "_moonep_weight_layout", None) is not None:
|
||||
# On a reload the layer parameters were already replaced by the
|
||||
# converted [E+B] gate/down tensors, so the source-format weights
|
||||
# this conversion needs are gone (and the unregistered up
|
||||
# projection cannot be recovered from them at all). Tracked in
|
||||
# RFC #52095.
|
||||
raise NotImplementedError(
|
||||
"MoonEP does not support in-place weight reloads yet"
|
||||
)
|
||||
layout = gather_moonep_weight_layout(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
num_global_experts=moe_config.num_experts,
|
||||
num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS,
|
||||
)
|
||||
layer._moonep_weight_layout = layout
|
||||
return layout.full_gate_weight, layout.full_down_weight
|
||||
|
||||
if unquantized_backend == UnquantizedMoeBackend.AITER:
|
||||
w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight)
|
||||
w13_weight.is_shuffled = True
|
||||
@@ -522,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]):
|
||||
moe_config: FusedMoEConfig,
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
layer: torch.nn.Module | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return convert_to_unquantized_kernel_format(
|
||||
backend, moe_config, w13_weight, w2_weight
|
||||
backend, moe_config, w13_weight, w2_weight, layer=layer
|
||||
)
|
||||
|
||||
def make_kernel(
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize.
|
||||
|
||||
BF16 correctness-first proof of concept on top of vLLM's modular kernel
|
||||
interface.
|
||||
|
||||
MoonEP differs from DeepEP-style backends in two ways that shape this
|
||||
integration:
|
||||
|
||||
- ``dispatch`` returns tokens already grouped by expert *row* (a fixed
|
||||
``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together
|
||||
with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is
|
||||
no per-token topk id tensor after dispatch; the expert compute must be a
|
||||
grouped GEMM over ``cu_seqlens`` segments.
|
||||
- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert
|
||||
prefetch slots. ``plan.experts_to_copy`` names the source expert of each
|
||||
slot and ``Buffer.prefetch_weight`` must run between dispatch and expert
|
||||
compute.
|
||||
|
||||
PoC limitations:
|
||||
- BF16 / unquantized only, eager only.
|
||||
- Expert weights are replicated in global-expert order on every rank
|
||||
(memory-heavy). Production Kimi-K3 serving requires sharded
|
||||
symmetric-memory expert ownership, where rows ``[0, E)`` physically alias
|
||||
each home rank's parameter memory.
|
||||
- Route weights are applied inside the expert compute and MoonEP's
|
||||
``combine`` performs the K-sum, so ``finalize`` requires
|
||||
``TopKWeightAndReduceNoOP``.
|
||||
"""
|
||||
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
|
||||
MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4
|
||||
MOONEP_DEFAULT_TOKEN_PADDING = 128
|
||||
MOONEP_DEFAULT_NUM_SMS = 32
|
||||
# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128.
|
||||
MOONEP_WEIGHT_TILE = 128
|
||||
|
||||
|
||||
class MoonEPExpertWeightLayout(NamedTuple):
|
||||
"""Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout.
|
||||
|
||||
Rows ``[0, E)`` hold expert weights in global expert order; rows
|
||||
``[E, E+B)`` are mutable prefetch slots filled by
|
||||
``Buffer.prefetch_weight``.
|
||||
|
||||
The prefetch slots never need re-zeroing between calls: the planner
|
||||
marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by
|
||||
``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so
|
||||
stale slot contents are never read; used slots are fully overwritten.
|
||||
"""
|
||||
|
||||
full_gate_weight: torch.Tensor
|
||||
full_up_weight: torch.Tensor
|
||||
full_down_weight: torch.Tensor
|
||||
num_prefetch_slots: int
|
||||
|
||||
|
||||
def make_moonep_weight_layout(
|
||||
w13_weight: torch.Tensor,
|
||||
w2_weight: torch.Tensor,
|
||||
num_prefetch_slots: int,
|
||||
) -> MoonEPExpertWeightLayout:
|
||||
"""Build the replicated ``[E+B, ...]`` PoC weight layout.
|
||||
|
||||
``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight``
|
||||
``[E, H, I]``, both BF16 in global expert order.
|
||||
"""
|
||||
if num_prefetch_slots <= 0:
|
||||
raise ValueError(
|
||||
f"num_prefetch_slots must be positive, got {num_prefetch_slots}"
|
||||
)
|
||||
if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("MoonEP PoC supports BF16 weights only.")
|
||||
num_experts, two_i, hidden_size = w13_weight.shape
|
||||
intermediate_size = two_i // 2
|
||||
if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size):
|
||||
raise ValueError(
|
||||
f"w2_weight shape {tuple(w2_weight.shape)} does not match "
|
||||
f"w13_weight shape {tuple(w13_weight.shape)}"
|
||||
)
|
||||
if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE:
|
||||
raise ValueError(
|
||||
"MoonEP weight prefetch requires hidden_size and intermediate_size "
|
||||
f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, "
|
||||
f"I={intermediate_size}"
|
||||
)
|
||||
|
||||
full_gate_weight = torch.empty(
|
||||
num_experts + num_prefetch_slots,
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=w13_weight.device,
|
||||
)
|
||||
full_up_weight = torch.empty_like(full_gate_weight)
|
||||
full_down_weight = torch.empty(
|
||||
num_experts + num_prefetch_slots,
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=w2_weight.device,
|
||||
)
|
||||
|
||||
full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :])
|
||||
full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :])
|
||||
full_down_weight[:num_experts].copy_(w2_weight)
|
||||
full_gate_weight[num_experts:].zero_()
|
||||
full_up_weight[num_experts:].zero_()
|
||||
full_down_weight[num_experts:].zero_()
|
||||
|
||||
return MoonEPExpertWeightLayout(
|
||||
full_gate_weight=full_gate_weight.contiguous(),
|
||||
full_up_weight=full_up_weight.contiguous(),
|
||||
full_down_weight=full_down_weight.contiguous(),
|
||||
num_prefetch_slots=num_prefetch_slots,
|
||||
)
|
||||
|
||||
|
||||
def gather_moonep_weight_layout(
|
||||
w13_local: torch.Tensor,
|
||||
w2_local: torch.Tensor,
|
||||
num_global_experts: int,
|
||||
num_prefetch_slots: int,
|
||||
) -> MoonEPExpertWeightLayout:
|
||||
"""Build the replicated ``[E+B, ...]`` layout from this rank's local experts.
|
||||
|
||||
PoC bridge: each EP rank loads only its own experts (linear placement),
|
||||
so all-gather them once at load time into global expert order on every
|
||||
rank. Production MoonEP instead maps rows ``[0, E)`` onto each home
|
||||
rank's parameter memory via symmetric memory (RFC #52095 item 5).
|
||||
"""
|
||||
from vllm.distributed import get_ep_group
|
||||
|
||||
ep_group = get_ep_group()
|
||||
ep_size = ep_group.world_size
|
||||
if ep_size == 1:
|
||||
return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots)
|
||||
if w13_local.size(0) * ep_size != num_global_experts:
|
||||
raise NotImplementedError(
|
||||
"MoonEP PoC requires num_experts to be evenly divisible across EP "
|
||||
f"ranks: {num_global_experts} experts, {ep_size} ranks"
|
||||
)
|
||||
w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0)
|
||||
w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0)
|
||||
return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots)
|
||||
|
||||
|
||||
MOONEP_MIN_DISPATCH_TOKENS = 128
|
||||
|
||||
|
||||
class MoonEPBufferPool:
|
||||
"""Lazily creates ``moonep.Buffer`` instances at power-of-two capacities.
|
||||
|
||||
``Buffer`` fixes its token capacity ``S`` at construction, so a single
|
||||
full-capacity buffer would pad every dispatch to
|
||||
``max_num_batched_tokens``. The pool instead serves the smallest
|
||||
power-of-two capacity that fits the step's token count, so decode-sized
|
||||
batches dispatch a few hundred slots rather than the full capacity.
|
||||
|
||||
Selection must be identical on every EP rank (dispatch is collective and
|
||||
``Buffer`` construction is itself a collective): callers derive the token
|
||||
count from ``dp_metadata`` (the max across DP ranks) so all ranks create
|
||||
and pick the same buffer at the same step.
|
||||
"""
|
||||
|
||||
def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int):
|
||||
self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"}
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self._buffers: dict[int, Any] = {}
|
||||
|
||||
def capacity_for(self, num_tokens: int) -> int:
|
||||
if num_tokens >= self.max_tokens_per_rank:
|
||||
return self.max_tokens_per_rank
|
||||
cap = 1 << max(num_tokens - 1, 0).bit_length()
|
||||
return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank)
|
||||
|
||||
def get(self, num_tokens: int) -> tuple[int, Any]:
|
||||
"""Return ``(capacity, buffer)`` for the given step token count."""
|
||||
from moonep import Buffer # type: ignore[import-not-found]
|
||||
|
||||
capacity = self.capacity_for(num_tokens)
|
||||
buffer = self._buffers.get(capacity)
|
||||
if buffer is None:
|
||||
buffer = Buffer(S=capacity, **self._kwargs)
|
||||
self._buffers[capacity] = buffer
|
||||
return capacity, buffer
|
||||
|
||||
def destroy(self) -> None:
|
||||
for buffer in self._buffers.values():
|
||||
buffer.destroy()
|
||||
self._buffers.clear()
|
||||
|
||||
|
||||
class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
"""Prepare/Finalize using MoonEP balanced dispatch/combine.
|
||||
|
||||
``prepare`` pads the batch to the selected buffer's static token
|
||||
capacity, dispatches, runs ``prefetch_weight`` for the planned redundant
|
||||
experts, and stashes the ``plan`` for ``finalize`` (the same pattern
|
||||
DeepEP-HT uses for its handle). Downstream expert compute must consume
|
||||
the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
buffer_pool: "MoonEPBufferPool",
|
||||
max_tokens_per_rank: int,
|
||||
num_dispatchers: int,
|
||||
num_global_experts: int,
|
||||
weight_layout: MoonEPExpertWeightLayout | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.buffer_pool = buffer_pool
|
||||
self.max_tokens_per_rank = max_tokens_per_rank
|
||||
self.num_dispatchers_ = num_dispatchers
|
||||
self.num_global_experts = num_global_experts
|
||||
self.weight_layout = weight_layout
|
||||
self._fused_experts: Any = None
|
||||
|
||||
# dispatch state consumed by finalize (and the expert runner)
|
||||
self._plan: Any = None
|
||||
self._cu_seqlens: torch.Tensor | None = None
|
||||
self._num_tokens: int = 0
|
||||
self._buffer: Any = None # the pool buffer used by the current step
|
||||
|
||||
def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None:
|
||||
# The [E+B] weight layout is attached to the experts by their
|
||||
# process_weights_after_loading hook (after this runs), so keep the
|
||||
# reference and resolve the layout lazily in prepare().
|
||||
self._fused_experts = fused_experts
|
||||
|
||||
def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout:
|
||||
if self.weight_layout is None and self._fused_experts is not None:
|
||||
layout = getattr(self._fused_experts, "weight_layout", None)
|
||||
if isinstance(layout, MoonEPExpertWeightLayout):
|
||||
self.weight_layout = layout
|
||||
# Redundant experts' weights must be in rows [E, E+B) before the
|
||||
# expert compute reads them; skipping prefetch silently corrupts
|
||||
# output.
|
||||
assert self.weight_layout is not None, (
|
||||
"MoonEPPrepareAndFinalize: weight layout not available (the "
|
||||
"experts' process_weights_after_loading has not run)"
|
||||
)
|
||||
return self.weight_layout
|
||||
|
||||
@property
|
||||
def num_dispatched_slots(self) -> int:
|
||||
"""``NvS`` of the buffer selected by the current step's prepare()."""
|
||||
assert self._buffer is not None, "prepare() has not been called"
|
||||
return int(self._buffer._ctx["NvS"])
|
||||
|
||||
def num_dispatchers(self) -> int:
|
||||
return self.num_dispatchers_
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
# combine returns the fully weighted+reduced token-major output
|
||||
return True
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
def max_num_tokens_per_rank(self) -> int | None:
|
||||
return self.max_tokens_per_rank
|
||||
|
||||
def topk_indices_dtype(self) -> torch.dtype | None:
|
||||
return torch.int32
|
||||
|
||||
def supports_async(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def cu_seqlens(self) -> torch.Tensor:
|
||||
assert self._cu_seqlens is not None, "prepare() has not been called"
|
||||
return self._cu_seqlens
|
||||
|
||||
@property
|
||||
def plan(self) -> Any:
|
||||
assert self._plan is not None, "prepare() has not been called"
|
||||
return self._plan
|
||||
|
||||
def _pad_to_capacity(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
capacity: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
num_tokens = a1.shape[0]
|
||||
if num_tokens > capacity:
|
||||
raise ValueError(
|
||||
f"MoonEP static buffer capacity exceeded: {num_tokens} tokens "
|
||||
f"> S={capacity}."
|
||||
)
|
||||
a1 = a1.contiguous()
|
||||
topk_ids = topk_ids.to(dtype=torch.int32)
|
||||
topk_weights = topk_weights.to(dtype=torch.float32)
|
||||
# MoonEP's planner has no sentinel for invalid expert ids, so map
|
||||
# negative ids (padded/invalid slots from the engine) to expert 0
|
||||
# with zero route weight — the same convention as the capacity
|
||||
# padding rows below. Wasted compute, never wrong output.
|
||||
invalid = topk_ids < 0
|
||||
topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids)
|
||||
topk_weights = torch.where(
|
||||
invalid, torch.zeros_like(topk_weights), topk_weights
|
||||
).contiguous()
|
||||
topk_ids = topk_ids.contiguous()
|
||||
if num_tokens == capacity:
|
||||
return a1, topk_ids, topk_weights, num_tokens
|
||||
pad = capacity - num_tokens
|
||||
return (
|
||||
F.pad(a1, (0, 0, 0, pad)),
|
||||
F.pad(topk_ids, (0, 0, 0, pad)),
|
||||
F.pad(topk_weights, (0, 0, 0, pad)),
|
||||
num_tokens,
|
||||
)
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
if a1.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.")
|
||||
if quant_config.quant_dtype is not None:
|
||||
raise NotImplementedError("MoonEP PoC does not support quantized dispatch.")
|
||||
assert num_experts == self.num_global_experts
|
||||
assert self._plan is None, (
|
||||
"MoonEPPrepareAndFinalize.prepare() called again before finalize()"
|
||||
)
|
||||
if apply_router_weight_on_input:
|
||||
assert topk_ids.size(1) == 1, (
|
||||
"apply_router_weight_on_input requires top-1 routing"
|
||||
)
|
||||
a1 = a1 * topk_weights.to(a1.dtype)
|
||||
|
||||
# Buffer choice must be rank-symmetric: use the max token count
|
||||
# across DP ranks (every rank sees the same dp_metadata), falling
|
||||
# back to the local count outside a forward context (tests).
|
||||
num_tokens = a1.shape[0]
|
||||
try:
|
||||
dp_meta = get_forward_context().dp_metadata
|
||||
except AssertionError:
|
||||
dp_meta = None
|
||||
if dp_meta is not None:
|
||||
num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max())
|
||||
capacity, self._buffer = self.buffer_pool.get(num_tokens)
|
||||
|
||||
a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity(
|
||||
a1, topk_ids, topk_weights, capacity
|
||||
)
|
||||
tokens_per_expert = torch.bincount(
|
||||
topk_ids.reshape(-1).to(dtype=torch.int64),
|
||||
minlength=num_experts,
|
||||
).to(dtype=torch.int32)
|
||||
|
||||
hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch(
|
||||
a1,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
tokens_per_expert,
|
||||
)
|
||||
self._plan = plan
|
||||
self._cu_seqlens = cu_seqlens
|
||||
|
||||
weight_layout = self._resolve_weight_layout()
|
||||
self._buffer.prefetch_weight(
|
||||
plan=plan,
|
||||
full_gate_weight=weight_layout.full_gate_weight,
|
||||
full_up_weight=weight_layout.full_up_weight,
|
||||
full_down_weight=weight_layout.full_down_weight,
|
||||
)
|
||||
|
||||
# Segment sizes per [E+B] weight row. NOTE: this is per *row*, not
|
||||
# per local expert, and NvS rows are expert-grouped rather than
|
||||
# token-major — only MoonEP-aware expert implementations can consume
|
||||
# this activation layout.
|
||||
expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1))
|
||||
expert_tokens_meta = mk.ExpertTokensMetadata(
|
||||
expert_num_tokens=expert_num_tokens,
|
||||
expert_num_tokens_cpu=None,
|
||||
)
|
||||
|
||||
# MoonEP has no post-dispatch per-token topk ids/weights; route
|
||||
# weights come back in NvS order.
|
||||
return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs
|
||||
|
||||
def finalize(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
fused_expert_output: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
weight_and_reduce_impl: mk.TopKWeightAndReduce,
|
||||
) -> None:
|
||||
# Route weights are applied in the expert compute and MoonEP's
|
||||
# combine performs the K-sum, so any weight/reduce work in finalize
|
||||
# means MoonEP was paired with the wrong kind of experts.
|
||||
assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), (
|
||||
"MoonEP requires TopKWeightAndReduceNoOP, got "
|
||||
f"{type(weight_and_reduce_impl).__name__}"
|
||||
)
|
||||
combined, _, _ = self._buffer.combine(
|
||||
plan=self.plan,
|
||||
hidden_nvsh=fused_expert_output,
|
||||
route_weights_nvs=None,
|
||||
)
|
||||
output.copy_(combined[: self._num_tokens])
|
||||
self._plan = None
|
||||
self._cu_seqlens = None
|
||||
@@ -49,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
|
||||
# --8<-- [end:unquantized_fused_moe]
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend(
|
||||
moe_config=self.moe,
|
||||
)
|
||||
# MoonEP replaces the named parameters with its [E+B] gate/down
|
||||
# split and keeps the up projection outside the parameter registry,
|
||||
# so a pre-processed (e.g. ipc_cache) load cannot reconstruct it.
|
||||
self.supports_pre_processed_weights = (
|
||||
self.unquantized_backend != UnquantizedMoeBackend.MOONEP
|
||||
)
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
@@ -171,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
moe_config=layer.moe_config,
|
||||
w13_weight=w13,
|
||||
w2_weight=w2,
|
||||
layer=layer,
|
||||
)
|
||||
# `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__;
|
||||
# On the first call we replace the parameter normally. On subsequent
|
||||
@@ -191,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
# does not need to be re-built.
|
||||
self._init_moe_kernel(layer)
|
||||
|
||||
if self.unquantized_backend == UnquantizedMoeBackend.CPU:
|
||||
# The CPU experts need the layer itself for the setup that
|
||||
# convert_to_unquantized_kernel_format cannot express, since
|
||||
# it only sees the two weight tensors: padding and prepacking
|
||||
# into the grouped-gemm layout (bias included), and capturing
|
||||
# the router config that monolithic apply() cannot carry.
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
# No-op by default. Experts that need the layer itself for setup
|
||||
# convert_to_unquantized_kernel_format cannot express override
|
||||
# it: CPU prepacks into its grouped-gemm layout and captures the
|
||||
# router config; MoonEP picks up the [E+B] weight layout for
|
||||
# prefetch and the up projection.
|
||||
assert self.moe_kernel is not None
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def _init_moe_kernel(self, layer: "RoutedExperts") -> None:
|
||||
"""Build the MoE kernel from the layer's current (shuffled) weights."""
|
||||
|
||||
@@ -512,6 +512,34 @@ def has_nixl_ep() -> bool:
|
||||
return _has_module("nixl_ep")
|
||||
|
||||
|
||||
def has_moonep() -> bool:
|
||||
"""Whether the optional `moonep` package is available."""
|
||||
return _has_module("moonep")
|
||||
|
||||
|
||||
def check_moonep_system_support() -> None:
|
||||
"""Raise if the current device cannot run MoonEP.
|
||||
|
||||
MoonEP's symmetric-memory buffers require CUDA VMM plus NVSwitch multicast
|
||||
(SHARP) on every EP rank; NVLink-only topologies without NVSwitch (e.g.
|
||||
4x H100 NV6) fail deep inside ``moonep.Buffer`` otherwise.
|
||||
"""
|
||||
if not has_moonep():
|
||||
raise RuntimeError(
|
||||
"MoonEP not available. Install it from "
|
||||
"https://github.com/MoonshotAI/MoonEP."
|
||||
)
|
||||
from moonep._C import ( # type: ignore[import-not-found]
|
||||
nvl_multicast_supported,
|
||||
)
|
||||
|
||||
if not nvl_multicast_supported():
|
||||
raise RuntimeError(
|
||||
"MoonEP requires NVSwitch multicast (SHARP) support on the "
|
||||
"current device; it is not supported on this GPU/topology."
|
||||
)
|
||||
|
||||
|
||||
def is_numba_available() -> bool:
|
||||
"""Whether the optional `numba` package is available."""
|
||||
return _has_module("numba")
|
||||
|
||||
Reference in New Issue
Block a user