diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml new file mode 100644 index 0000000000..8704f29bcf --- /dev/null +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-BF16-moonep.yaml @@ -0,0 +1,5 @@ +model_name: "Qwen/Qwen3-30B-A3B" +accuracy_threshold: 0.88 +num_questions: 1319 +num_fewshot: 5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend=moonep" diff --git a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt index 7fb1f49685..6dcb7414a5 100644 --- a/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt +++ b/tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt @@ -10,3 +10,4 @@ Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml Qwen3-30B-A3B-BF16-triton.yaml +Qwen3-30B-A3B-BF16-moonep.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 477a69a16c..7411ca0f40 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -128,6 +128,12 @@ def test_gsm8k_correctness(config_filename): "Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. " "The quantization scheme is not supported on non-GFX950 platforms." ) + if "--all2all-backend=moonep" in eval_config.get("server_args", ""): + from vllm.utils.import_utils import has_moonep + + if not has_moonep(): + pytest.skip("Skipping MoonEP config: the moonep package is not installed.") + # Parse server arguments from config (use shlex to handle quoted strings) server_args_str = eval_config.get("server_args", "") server_args = shlex.split(server_args_str) if server_args_str else [] diff --git a/tests/kernels/moe/test_moonep_bf16_poc.py b/tests/kernels/moe/test_moonep_bf16_poc.py new file mode 100644 index 0000000000..ba1e78fb7b --- /dev/null +++ b/tests/kernels/moe/test_moonep_bf16_poc.py @@ -0,0 +1,412 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Test MoonEP dispatch / prefetch / combine logic (BF16 PoC). + +Two paths, both compared against the pure-PyTorch reference MoE: +- MoonEPPrepareAndFinalize + a reference segment-loop expert runner over + MoonEP's expert-grouped ``[NvS, H]`` layout; +- the full modular kernel, MoonEPPrepareAndFinalize + MoonEPExperts through + FusedMoEKernel. +Requires NVSwitch multicast capable GPUs. +""" + +import dataclasses +import types + +import pytest +import torch +import torch.nn.functional as F +from torch.distributed import ProcessGroup + +from tests.kernels.moe.utils import make_test_weights +from tests.kernels.utils import torch_experts +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.utils.import_utils import has_moonep +from vllm.utils.torch_utils import set_random_seed + +from ...utils import multi_gpu_test +from .parallel_utils import ProcessGroupInfo, parallel_launch + +if has_moonep(): + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPExpertWeightLayout, + MoonEPPrepareAndFinalize, + make_moonep_weight_layout, + ) + +requires_moonep = pytest.mark.skipif( + not has_moonep(), + reason="Requires MoonEP", +) + + +class MulticastNotAvailableError(RuntimeError): + pass + + +@dataclasses.dataclass +class TestConfig: + topk: int + m: int + k: int + n: int + num_experts: int + router_skew: float + apply_router_weight_on_input: bool = False + + +@dataclasses.dataclass +class TestTensors: + rank_tokens: torch.Tensor + topk: torch.Tensor + topk_weights: torch.Tensor + config: TestConfig + + @staticmethod + def make(config: TestConfig) -> "TestTensors": + rank_tokens = ( + torch.randn((config.m, config.k), device="cuda", dtype=torch.bfloat16) / 10 + ) + # Skewed routing via a per-expert bias: scaling logits by a scalar + # preserves their ordering and would not change the selected topk + # ids, but a shared bias makes a few experts hot on every token so + # the planner has to fill redundant-expert prefetch slots. + expert_bias = config.router_skew * torch.randn( + 1, config.num_experts, device="cuda", dtype=torch.float32 + ) + logits = ( + torch.randn( + config.m, config.num_experts, device="cuda", dtype=torch.float32 + ) + + expert_bias + ) + topk_weights, topk = torch.topk(logits, config.topk, dim=-1) + topk_weights = torch.softmax(topk_weights, dim=-1) + return TestTensors( + rank_tokens=rank_tokens, + topk=topk.to(dtype=torch.int64), + topk_weights=topk_weights, + config=config, + ) + + +def reference_moonep_experts( + hidden_nvsh: torch.Tensor, + route_weights_nvs: torch.Tensor, + cu_seqlens: torch.Tensor, + weight_layout: "MoonEPExpertWeightLayout", +) -> torch.Tensor: + """Segment loop over MoonEP's expert-grouped layout. + + ``prefetch_weight`` has already materialized redundant experts' weights + in rows ``[E, E+B)``, so every segment reads its own row. Route weights + are applied here; MoonEP's combine does the K-sum. + """ + output = torch.empty_like(hidden_nvsh) + prev = 0 + for row, cur in enumerate(cu_seqlens.tolist()): + if cur == prev: + continue + x = hidden_nvsh[prev:cur] + gate = F.linear(x, weight_layout.full_gate_weight[row]) + up = F.linear(x, weight_layout.full_up_weight[row]) + y = F.linear(F.silu(gate) * up, weight_layout.full_down_weight[row]) + y = y * route_weights_nvs[prev:cur].to(dtype=y.dtype).unsqueeze(-1) + output[prev:cur].copy_(y) + prev = cur + if prev < hidden_nvsh.shape[0]: + output[prev:].zero_() + return output + + +def make_moonep_prepare_finalize( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + hidden_size: int, + num_experts: int, + topk: int, + max_tokens_per_rank: int, + weight_layout: "MoonEPExpertWeightLayout", + pass_layout_to_pf: bool = True, +): + from moonep._C import nvl_multicast_supported + + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPBufferPool, + ) + + if not nvl_multicast_supported(): + raise MulticastNotAvailableError("NVSwitch multicast not available") + + pool = MoonEPBufferPool( + dict( + H=hidden_size, + K=topk, + E=num_experts, + num_ep_ranks=pgi.world_size, + B=weight_layout.num_prefetch_slots, + group=pg, + explicitly_destroy=True, + ), + max_tokens_per_rank=max_tokens_per_rank, + ) + return pool, MoonEPPrepareAndFinalize( + buffer_pool=pool, + max_tokens_per_rank=max_tokens_per_rank, + num_dispatchers=pgi.world_size, + num_global_experts=num_experts, + weight_layout=weight_layout if pass_layout_to_pf else None, + ) + + +def moonep_moe_impl( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + test_tensors: TestTensors, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, +) -> torch.Tensor: + config = test_tensors.config + hidden_size = test_tensors.rank_tokens.shape[1] + max_tokens_per_rank = 128 * ((config.m + 127) // 128) + + weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots) + pool, pf = make_moonep_prepare_finalize( + pg, + pgi, + hidden_size, + config.num_experts, + config.topk, + max_tokens_per_rank, + weight_layout, + ) + try: + hidden_nvsh, _, _, _, route_weights_nvs = pf.prepare( + test_tensors.rank_tokens, + test_tensors.topk_weights, + test_tensors.topk, + num_experts=config.num_experts, + expert_map=None, + apply_router_weight_on_input=config.apply_router_weight_on_input, + quant_config=_no_quant_config(), + ) + assert route_weights_nvs is not None + if config.apply_router_weight_on_input: + # prepare() already applied the weights to the inputs; the + # runner must not apply them again. + route_weights_nvs = torch.ones_like(route_weights_nvs) + if config.router_skew >= 8 and config.m >= 100: + # Heavy skew must actually engage the redundant-expert planner. + assert (pf.plan.experts_to_copy >= 0).any(), ( + "no prefetch slot used despite heavy router skew" + ) + expert_out = reference_moonep_experts( + hidden_nvsh, route_weights_nvs, pf.cu_seqlens, weight_layout + ) + output = torch.empty_like(test_tensors.rank_tokens) + pf.finalize( + output, + expert_out, + test_tensors.topk_weights, + test_tensors.topk, + apply_router_weight_on_input=False, + weight_and_reduce_impl=TopKWeightAndReduceNoOP(), + ) + torch.accelerator.synchronize() + return output + finally: + pool.destroy() + + +def _no_quant_config(): + from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig + + return FusedMoEQuantConfig.make(quant_dtype=None) + + +def moonep_modular_kernel_impl( + pg: ProcessGroup, + pgi: ProcessGroupInfo, + test_tensors: TestTensors, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, +) -> torch.Tensor: + """Full modular-kernel path: MoonEPPrepareAndFinalize + MoonEPExperts.""" + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.experts.moonep_experts import ( + MoonEPExperts, + ) + from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel + + config = test_tensors.config + hidden_size = test_tensors.rank_tokens.shape[1] + max_tokens_per_rank = 128 * ((config.m + 127) // 128) + + weight_layout = make_moonep_weight_layout(w1, w2, num_prefetch_slots) + # weight_layout deliberately not passed to the P/F: the engine path + # resolves it through post_init_setup + the experts' weight hook. + pool, pf = make_moonep_prepare_finalize( + pg, + pgi, + hidden_size, + config.num_experts, + config.topk, + max_tokens_per_rank, + weight_layout, + pass_layout_to_pf=False, + ) + try: + moe_config = make_dummy_moe_config( + num_experts=config.num_experts, + experts_per_token=config.topk, + hidden_dim=hidden_size, + intermediate_size=config.n, + max_num_tokens=max_tokens_per_rank, + ) + experts = MoonEPExperts(moe_config=moe_config, quant_config=_no_quant_config()) + # Stand-in for the layer carrying the layout, as + # convert_to_unquantized_kernel_format leaves it in the engine path. + fake_layer = types.SimpleNamespace(_moonep_weight_layout=weight_layout) + experts.process_weights_after_loading(fake_layer) + kernel = FusedMoEKernel(prepare_finalize=pf, fused_experts=experts) + out = kernel.apply( + hidden_states=test_tensors.rank_tokens, + w1=weight_layout.full_gate_weight, + w2=weight_layout.full_down_weight, + topk_weights=test_tensors.topk_weights, + topk_ids=test_tensors.topk, + activation=MoEActivation.SILU, + global_num_experts=config.num_experts, + expert_map=None, + apply_router_weight_on_input=config.apply_router_weight_on_input, + ) + torch.accelerator.synchronize() + return out + finally: + pool.destroy() + + +def _moonep_moe( + pgi: ProcessGroupInfo, + config: TestConfig, + w1: torch.Tensor, + w2: torch.Tensor, + num_prefetch_slots: int, + use_modular_kernel: bool, +): + from vllm.v1.worker.workspace import init_workspace_manager + + device_idx = torch.accelerator.current_device_index() + init_workspace_manager(torch.device("cuda", device_idx)) + w1 = w1.to(device=device_idx) + w2 = w2.to(device=device_idx) + + pg = torch.distributed.new_group(list(range(pgi.world_size))) + set_random_seed(7 + pgi.rank) + test_tensors = TestTensors.make(config) + + with set_current_vllm_config(VllmConfig()): + torch_combined = torch_experts( + test_tensors.rank_tokens, + w1, + w2, + test_tensors.topk_weights, + test_tensors.topk, + apply_router_weights_on_input=config.apply_router_weight_on_input, + ) + impl = moonep_modular_kernel_impl if use_modular_kernel else moonep_moe_impl + moonep_combined = impl(pg, pgi, test_tensors, w1, w2, num_prefetch_slots) + + torch.testing.assert_close( + torch_combined, + moonep_combined, + atol=6e-2, + rtol=6e-2, + ) + + +MNKs = [ + (1, 256, 512), + (37, 256, 512), + (100, 512, 1024), + (512, 768, 2048), +] + + +@pytest.mark.parametrize("m,n,k", MNKs) +@pytest.mark.parametrize("num_experts", [32]) +@pytest.mark.parametrize("topk", [4]) +@pytest.mark.parametrize("router_skew", [1.0, 8.0]) +@pytest.mark.parametrize("num_prefetch_slots", [4]) +@pytest.mark.parametrize("world_size", [2]) +@pytest.mark.parametrize("use_modular_kernel", [False, True]) +@multi_gpu_test(num_gpus=2) +@requires_moonep +def test_moonep_bf16_moe( + m: int, + n: int, + k: int, + num_experts: int, + topk: int, + router_skew: float, + num_prefetch_slots: int, + world_size: int, + use_modular_kernel: bool, +): + set_random_seed(7) + config = TestConfig( + topk=topk, + m=m, + k=k, + n=n, + num_experts=num_experts, + router_skew=router_skew, + ) + (_, w1, _, _), (_, w2, _, _) = make_test_weights(num_experts, n, k) + + try: + parallel_launch( + world_size, + _moonep_moe, + config, + w1, + w2, + num_prefetch_slots, + use_modular_kernel, + ) + except Exception as exc: + if "MulticastNotAvailableError" in str(exc): + pytest.skip("NVSwitch multicast not available") + raise + + +@pytest.mark.parametrize("use_modular_kernel", [False, True]) +@multi_gpu_test(num_gpus=2) +@requires_moonep +def test_moonep_input_weighted_moe(use_modular_kernel: bool): + """Llama-4-style routing applies route weights to the expert inputs.""" + set_random_seed(7) + config = TestConfig( + topk=1, + m=100, + k=1024, + n=512, + num_experts=32, + router_skew=1.0, + apply_router_weight_on_input=True, + ) + (_, w1, _, _), (_, w2, _, _) = make_test_weights( + config.num_experts, config.n, config.k + ) + try: + parallel_launch(2, _moonep_moe, config, w1, w2, 4, use_modular_kernel) + except Exception as exc: + if "MulticastNotAvailableError" in str(exc): + pytest.skip("NVSwitch multicast not available") + raise diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index ba744e049f..666f2441ae 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -49,6 +49,7 @@ All2AllBackend = Literal[ "deepep_v2", "mori_high_throughput", "mori_low_latency", + "moonep", "nixl_ep", "allgather_reducescatter", "flashinfer_all2allv", # temporary alias for flashinfer_nvlink_two_sided @@ -206,6 +207,7 @@ class ParallelConfig: - "deepep_low_latency": Use deepep low-latency kernels - "mori_high_throughput": MoRI EP with InterNodeV1 for multi-node - "mori_low_latency": MoRI EP with InterNodeV1LL for multi-node + - "moonep": MoonEP balanced EP with dynamic redundant experts (NVLink) - "nixl_ep": Use nixl-ep kernels - "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels - "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 4431743a3d..2ee458e7ff 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1881,6 +1881,55 @@ class VllmConfig: # (e.g., XPU may lower max_num_batched_tokens when MLA is enabled) self._set_compile_ranges() + if self.parallel_config.all2all_backend == "moonep": + if ( + self.model_config is not None + and self.model_config.quantization is not None + ): + raise ValueError( + "The moonep all2all backend currently supports unquantized " + "BF16 models only; got " + f"quantization={self.model_config.quantization!r}. Use a " + "different --all2all-backend for quantized models." + ) + if ( + self.model_config is not None + and self.model_config.dtype != torch.bfloat16 + ): + raise ValueError( + "The moonep all2all backend currently supports BF16 models " + f"only; got dtype={self.model_config.dtype}. Use a " + "different --all2all-backend or --dtype bfloat16." + ) + if self.parallel_config.enable_eplb: + raise ValueError( + "The moonep all2all backend does not support EPLB yet: " + "EPLB rearranges expert parameters in a layout MoonEP's " + "replicated [E+B] weights do not follow. Disable " + "--enable-eplb or use a different --all2all-backend." + ) + if self.parallel_config.expert_placement_strategy != "linear": + raise ValueError( + "The moonep all2all backend requires linear expert " + "placement: its load-time all-gather assumes each rank " + "holds a contiguous chunk of the global expert range. Got " + "--expert-placement-strategy " + f"{self.parallel_config.expert_placement_strategy!r}." + ) + # Enforced here rather than in set_splitting_ops_for_v1 so it + # holds for every compilation mode, and keyed on use_all2all so + # PCP/SP-only topologies are covered too: MoonEP dispatch/combine + # are eager-only and must not be captured. + if ( + self.parallel_config.use_all2all + and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE + ): + logger.info( + "MoonEP: Disabling CUDA Graphs since the MoonEP " + "integration is currently eager-only." + ) + self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE + # Do this after all the updates to compilation_config.mode effective_dp_size = ( self.parallel_config.data_parallel_size diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 0c8ebf3792..9e53c12e24 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -21,7 +21,12 @@ from vllm.utils.flashinfer import ( has_flashinfer_nvlink_two_sided, ) from vllm.utils.func_utils import supports_kw -from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2, has_mori +from vllm.utils.import_utils import ( + check_moonep_system_support, + has_deep_ep, + has_deep_ep_v2, + has_mori, +) from .base_device_communicator import All2AllManagerBase, Cache @@ -1094,3 +1099,71 @@ class DeepEPV2All2AllManager(All2AllManagerBase): for _, handle in self.handle_cache._cache.items(): handle.destroy() self.handle_cache._cache.clear() + + +class MoonEPAll2AllManager(All2AllManagerBase): + """All2All communication based on MoonEP + (https://github.com/MoonshotAI/MoonEP). + + MoonEP keeps token loads perfectly balanced across EP ranks by planning + a small number of dynamically redundant experts online and prefetching + their weights before expert compute. Every rank receives exactly + S x K token slots regardless of router skew, so all communication and + compute shapes are static. + + Requires NVLink symmetric-memory / multicast capable topologies + (single node NVSwitch, e.g. H100/H200/B200/GB300 class). + """ + + def __init__(self, cpu_group, tcp_store_group=None, device_group=None): + check_moonep_system_support() + super().__init__(cpu_group, tcp_store_group) + self._device_group = device_group + self.handle_cache = Cache() + + def _make_buffer_kwargs( + self, + max_num_tokens_per_dp_rank: int, + token_hidden_size: int, + num_topk: int, + num_global_experts: int, + num_prefetch_slots: int, + token_padding: int, + num_sms: int, + ) -> dict: + return dict( + S=max_num_tokens_per_dp_rank, + H=token_hidden_size, + K=num_topk, + E=num_global_experts, + num_ep_ranks=self.world_size, + num_sms=num_sms, + token_padding=token_padding, + B=num_prefetch_slots, + group=self._device_group + if self._device_group is not None + else self.cpu_group, + explicitly_destroy=True, + ) + + def get_handle(self, kwargs): + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MoonEPBufferPool, + ) + + buffer_kwargs = self._make_buffer_kwargs(**kwargs) + logger.debug("MoonEP all2all args %s", buffer_kwargs) + + def make_pool(**kw): + return MoonEPBufferPool(kw, max_tokens_per_rank=kw["S"]) + + handle: MoonEPBufferPool = self.handle_cache.get_or_create( + buffer_kwargs, make_pool + ) + return handle + + def destroy(self): + with self.handle_cache._lock: + for _, handle in self.handle_cache._cache.items(): + handle.destroy() + self.handle_cache._cache.clear() diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index fb4ccd9f1e..fb0c281b4a 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -214,6 +214,14 @@ class CudaCommunicator(DeviceCommunicatorBase): tcp_store_group, device_group=self.device_group, ) + elif self.all2all_backend == "moonep": + from .all2all import MoonEPAll2AllManager + + self.all2all_manager = MoonEPAll2AllManager( + self.cpu_group, + tcp_store_group, + device_group=self.device_group, + ) elif self.all2all_backend == "nixl_ep": from .all2all import NixlEPAll2AllManager diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 775f595bfa..eb7edaef53 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -35,6 +35,7 @@ from vllm.platforms import current_platform from vllm.utils.import_utils import ( has_deep_ep, has_deep_ep_v2, + has_moonep, has_mori, has_nixl_ep, ) @@ -103,6 +104,13 @@ if current_platform.is_cuda_alike(): NIXL_EP_QUANT_BLOCK_SHAPE, NixlEPPrepareAndFinalize, ) + if has_moonep(): + from .prepare_finalize.moonep import ( + MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + MOONEP_DEFAULT_NUM_SMS, + MOONEP_DEFAULT_TOKEN_PADDING, + MoonEPPrepareAndFinalize, + ) def get_ep_all2all_manager( @@ -279,6 +287,28 @@ def maybe_make_prepare_finalize( sp_size=moe.moe_parallel_config.sp_size, ) + elif moe.use_moonep_kernels: + all_to_all_args = dict( + max_num_tokens_per_dp_rank=moe.max_num_tokens, + token_hidden_size=moe.hidden_dim, + num_topk=moe.experts_per_token, + num_global_experts=moe.num_experts, + num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + token_padding=MOONEP_DEFAULT_TOKEN_PADDING, + num_sms=MOONEP_DEFAULT_NUM_SMS, + ) + handle = all2all_manager.get_handle(all_to_all_args) + + # The [E+B] weight layout is picked up from the experts (their + # process_weights_after_loading hook) once the layer has loaded and + # converted its weights. + prepare_finalize = MoonEPPrepareAndFinalize( + buffer_pool=handle, + max_tokens_per_rank=moe.max_num_tokens, + num_dispatchers=all2all_manager.world_size, + num_global_experts=moe.num_experts, + ) + elif moe.use_mori_kernels: assert quant_config is not None diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 35973e635c..62ab2bf88a 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -1085,6 +1085,10 @@ class FusedMoEParallelConfig: def use_deepep_v2_kernels(self): return self.use_all2all_kernels and self.all2all_backend == "deepep_v2" + @property + def use_moonep_kernels(self): + return self.use_all2all_kernels and self.all2all_backend == "moonep" + @staticmethod def flatten_tp_across_dp_and_pcp( tp_size: int, dp_size: int, dp_rank: int, pcp_size: int, pcp_rank: int @@ -1473,6 +1477,10 @@ class FusedMoEConfig: def use_deepep_v2_kernels(self): return self.moe_parallel_config.use_deepep_v2_kernels + @property + def use_moonep_kernels(self): + return self.moe_parallel_config.use_moonep_kernels + @property def needs_round_robin_routing_tables(self): return self.moe_parallel_config.needs_round_robin_routing_tables diff --git a/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py b/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py new file mode 100644 index 0000000000..1af4b5f064 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/moonep_experts.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MoonEP experts: grouped GEMM over MoonEP's expert-grouped activations. + +MoonEP dispatch delivers tokens already contiguous per weight row +(``[NvS, H]`` in ``cu_seqlens[E+B]`` segment order), so the expert compute +is three grouped GEMMs over those segments with no permute/unpermute: + + gate = grouped_mm(x, w_gate[E+B]) # [NvS, I] + up = grouped_mm(x, w_up[E+B]) # [NvS, I] + act = silu(gate) * up * route_weight # route weights applied here + out = grouped_mm(act, w_down[E+B]) # [NvS, H] + +Rows ``[E, E+B)`` are the redundant-expert prefetch slots that +``MoonEPPrepareAndFinalize`` fills before ``apply`` runs. Empty segments +(including unused prefetch slots) are skipped by the grouped GEMM. + +Weight layout: MoonEP's ``prefetch_weight`` requires each projection to be +its own contiguous ``[E+B, ., .]`` tensor, so gate and up are separate +tensors rather than vLLM's interleaved ``[E, 2I, H]`` ``w13``. The modular +kernel passes ``w1``/``w2`` from the layer (``w1`` is the gate tensor); the +full layout is picked up from the layer in ``process_weights_after_loading`` +and shared with ``MoonEPPrepareAndFinalize`` via ``post_init_setup``. +""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey +from vllm.platforms import current_platform + + +def moonep_grouped_gemm( + a: torch.Tensor, + w: torch.Tensor, + cu_seqlens: torch.Tensor, +) -> torch.Tensor: + """``out[seg_g] = a[seg_g] @ w[g].T`` for each ``cu_seqlens`` segment. + + Args: + a: ``[NvS, K]`` activations in segment order. + w: ``[G, N, K]`` weights (vLLM ``[E, N, K]`` convention). + cu_seqlens: ``[G]`` int32 cumulative segment end offsets. + + Returns: + ``[NvS, N]``; rows past ``cu_seqlens[-1]`` are zero-filled. + + """ + return torch._grouped_mm(a, w.transpose(1, 2), offs=cu_seqlens) + + +class MoonEPExperts(mk.FusedMoEExpertsModular): + """Grouped-GEMM experts over MoonEP's ``[NvS, H]`` layout (BF16).""" + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + max_num_tokens: int | None = None, + num_dispatchers: int | None = None, + ): + super().__init__(moe_config, quant_config, max_num_tokens, num_dispatchers) + assert quant_config.quant_dtype is None, ( + "MoonEPExperts supports unquantized BF16 only" + ) + if ( + quant_config.gemm1_alpha is not None + or quant_config.gemm1_beta is not None + or quant_config.gemm1_clamp_limit is not None + ): + raise NotImplementedError( + "MoonEPExperts implements plain silu(gate) * up only; SwiGLU " + "variants with alpha/beta/clamp parameters are not supported" + ) + self._weight_layout = None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + # The [E+B] gate/up/down layout built by + # convert_to_unquantized_kernel_format; MoonEPPrepareAndFinalize + # reads it from here (post_init_setup) for weight prefetch. + layout = getattr(layer, "_moonep_weight_layout", None) + assert layout is not None, ( + "MoonEPExperts requires the layer to carry _moonep_weight_layout " + "(set by convert_to_unquantized_kernel_format)" + ) + self._weight_layout = layout + + @property + def weight_layout(self): + return self._weight_layout + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return current_platform.is_cuda() + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation == MoEActivation.SILU + + @staticmethod + def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: + # EPLB rearranges the named expert parameters as local_num_experts + # rows, which does not understand the replicated [E+B] layout. + return ( + moe_parallel_config.use_moonep_kernels + and not moe_parallel_config.enable_eplb + ) + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return weight_key is None and activation_key is None + + def supports_chunking(self) -> bool: + # NvS is static and segments must stay whole. + return False + + def supports_expert_map(self) -> bool: + # MoonEP addresses global expert rows directly. + return False + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def moe_problem_size( + self, + a1: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_ids: torch.Tensor, + ) -> tuple[int, int, int, int, int]: + # a1 is [NvS, H] in segment order, not token-major, so the base + # implementation's topk_ids/a1 row-count check does not apply. + assert a1.dim() == 2 and w1.dim() == 3 and w2.dim() == 3 + num_rows, intermediate, hidden = w1.shape # [E+B, I, H] + assert a1.size(1) == hidden + return num_rows, a1.size(0), intermediate, hidden, topk_ids.size(1) + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # M is NvS. torch._grouped_mm allocates its own outputs, so no + # workspaces are needed. + return ((0,), (0,), (M, K)) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ) -> None: + # expert_map is ignored: MoonEP dispatches on global expert ids and + # the weights are addressed by global [E+B] row, on every rank. + assert activation == MoEActivation.SILU + assert self._weight_layout is not None, ( + "process_weights_after_loading() not called" + ) + assert expert_tokens_meta is not None + # MoonEPPrepareAndFinalize returns route weights in NvS order as the + # dispatched topk_weights, and per-row segment lengths as + # expert_tokens_meta. + route_weights_nvs = topk_weights + assert route_weights_nvs.dim() == 1 + cu_seqlens = torch.cumsum( + expert_tokens_meta.expert_num_tokens, dim=0, dtype=torch.int32 + ) + + assert cu_seqlens.numel() == w1.size(0) + + gate = moonep_grouped_gemm(hidden_states, w1, cu_seqlens) + up = moonep_grouped_gemm( + hidden_states, self._weight_layout.full_up_weight, cu_seqlens + ) + act = torch.nn.functional.silu(gate) + act.mul_(up) + if not apply_router_weight_on_input: + # Input-weighted routing (top-1, e.g. Llama 4) was already + # applied to the activations by prepare(), before dispatch. + act.mul_(route_weights_nvs.to(act.dtype).unsqueeze(-1)) + # Padding rows past the last segment come out zero-filled. + output.copy_(moonep_grouped_gemm(act, w2, cu_seqlens)) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index bcb00eecbb..b9f90012f4 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -40,6 +40,7 @@ class UnquantizedMoeBackend(Enum): AITER = "ROCm AITER" TRITON = "TRITON" BATCHED_TRITON = "BATCHED_TRITON" + MOONEP = "MOONEP" CPU = "CPU" XPU = "XPU" TPU = "TPU" @@ -138,6 +139,13 @@ def backend_to_kernel_cls( return [BatchedTritonExperts] + elif backend == UnquantizedMoeBackend.MOONEP: + from vllm.model_executor.layers.fused_moe.experts.moonep_experts import ( + MoonEPExperts, + ) + + return [MoonEPExperts] + elif backend == UnquantizedMoeBackend.XPU: from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts @@ -281,6 +289,13 @@ def select_unquantized_moe_backend( return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) + # MoonEP owns the expert layout (expert-grouped [NvS, H] + cu_seqlens), + # so it is not interchangeable with the token-major experts backends. + if moe_config.moe_parallel_config.use_moonep_kernels: + return _return_or_raise( + UnquantizedMoeBackend.MOONEP, moe_config, activation_format + ) + runner_backend = moe_config.moe_backend # 'humming' is quantization-only; an unquantized layer (e.g. excluded via # modules_to_not_convert) falls through to auto instead of erroring. @@ -366,7 +381,42 @@ def convert_to_unquantized_kernel_format( moe_config: FusedMoEConfig, w13_weight: torch.Tensor, w2_weight: torch.Tensor, + layer: torch.nn.Module | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: + if unquantized_backend == UnquantizedMoeBackend.MOONEP: + if layer is None: + raise ValueError( + "MoonEP weight conversion requires the layer module to stash " + "the [E+B] weight layout on" + ) + # MoonEP addresses expert weights by global [E+B] row and needs one + # contiguous tensor per projection (gate / up / down). The layer's + # w13_weight becomes the gate tensor and w2_weight the down tensor; + # the up tensor and the full layout are stashed on the layer for + # the kernel wiring in _setup_kernel. + from vllm.model_executor.layers.fused_moe.prepare_finalize.moonep import ( + MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + gather_moonep_weight_layout, + ) + + if getattr(layer, "_moonep_weight_layout", None) is not None: + # On a reload the layer parameters were already replaced by the + # converted [E+B] gate/down tensors, so the source-format weights + # this conversion needs are gone (and the unregistered up + # projection cannot be recovered from them at all). Tracked in + # RFC #52095. + raise NotImplementedError( + "MoonEP does not support in-place weight reloads yet" + ) + layout = gather_moonep_weight_layout( + w13_weight, + w2_weight, + num_global_experts=moe_config.num_experts, + num_prefetch_slots=MOONEP_DEFAULT_NUM_PREFETCH_SLOTS, + ) + layer._moonep_weight_layout = layout + return layout.full_gate_weight, layout.full_down_weight + if unquantized_backend == UnquantizedMoeBackend.AITER: w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight) w13_weight.is_shuffled = True @@ -522,9 +572,10 @@ class UnquantizedMoEKernelOracle(MoEKernelOracle[UnquantizedMoeBackend]): moe_config: FusedMoEConfig, w13_weight: torch.Tensor, w2_weight: torch.Tensor, + layer: torch.nn.Module | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: return convert_to_unquantized_kernel_format( - backend, moe_config, w13_weight, w2_weight + backend, moe_config, w13_weight, w2_weight, layer=layer ) def make_kernel( diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py new file mode 100644 index 0000000000..fdbed5ade8 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/moonep.py @@ -0,0 +1,429 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MoonEP (https://github.com/MoonshotAI/MoonEP) prepare/finalize. + +BF16 correctness-first proof of concept on top of vLLM's modular kernel +interface. + +MoonEP differs from DeepEP-style backends in two ways that shape this +integration: + +- ``dispatch`` returns tokens already grouped by expert *row* (a fixed + ``[NvS, H]`` layout with ``NvS = S x K`` real slots plus padding) together + with a ``cu_seqlens[E+B]`` segment table and an opaque ``plan``. There is + no per-token topk id tensor after dispatch; the expert compute must be a + grouped GEMM over ``cu_seqlens`` segments. +- Rows ``[E, E+B)`` of the weight/segment space are dynamic redundant-expert + prefetch slots. ``plan.experts_to_copy`` names the source expert of each + slot and ``Buffer.prefetch_weight`` must run between dispatch and expert + compute. + +PoC limitations: +- BF16 / unquantized only, eager only. +- Expert weights are replicated in global-expert order on every rank + (memory-heavy). Production Kimi-K3 serving requires sharded + symmetric-memory expert ownership, where rows ``[0, E)`` physically alias + each home rank's parameter memory. +- Route weights are applied inside the expert compute and MoonEP's + ``combine`` performs the K-sum, so ``finalize`` requires + ``TopKWeightAndReduceNoOP``. +""" + +from typing import Any, NamedTuple + +import torch +import torch.nn.functional as F + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) + +MOONEP_DEFAULT_NUM_PREFETCH_SLOTS = 4 +MOONEP_DEFAULT_TOKEN_PADDING = 128 +MOONEP_DEFAULT_NUM_SMS = 32 +# MoonEP's weight-prefetch kernel tiles hidden and intermediate dims by 128. +MOONEP_WEIGHT_TILE = 128 + + +class MoonEPExpertWeightLayout(NamedTuple): + """Contiguous BF16 expert weights in MoonEP ``[E+B, ...]`` layout. + + Rows ``[0, E)`` hold expert weights in global expert order; rows + ``[E, E+B)`` are mutable prefetch slots filled by + ``Buffer.prefetch_weight``. + + The prefetch slots never need re-zeroing between calls: the planner + marks unused slots with ``-1`` in ``plan.experts_to_copy`` (skipped by + ``prefetch_weight``) and gives them an empty ``cu_seqlens`` segment, so + stale slot contents are never read; used slots are fully overwritten. + """ + + full_gate_weight: torch.Tensor + full_up_weight: torch.Tensor + full_down_weight: torch.Tensor + num_prefetch_slots: int + + +def make_moonep_weight_layout( + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, + num_prefetch_slots: int, +) -> MoonEPExpertWeightLayout: + """Build the replicated ``[E+B, ...]`` PoC weight layout. + + ``w13_weight`` must be ``[E, 2I, H]`` (gate rows first) and ``w2_weight`` + ``[E, H, I]``, both BF16 in global expert order. + """ + if num_prefetch_slots <= 0: + raise ValueError( + f"num_prefetch_slots must be positive, got {num_prefetch_slots}" + ) + if w13_weight.dtype != torch.bfloat16 or w2_weight.dtype != torch.bfloat16: + raise NotImplementedError("MoonEP PoC supports BF16 weights only.") + num_experts, two_i, hidden_size = w13_weight.shape + intermediate_size = two_i // 2 + if tuple(w2_weight.shape) != (num_experts, hidden_size, intermediate_size): + raise ValueError( + f"w2_weight shape {tuple(w2_weight.shape)} does not match " + f"w13_weight shape {tuple(w13_weight.shape)}" + ) + if hidden_size % MOONEP_WEIGHT_TILE or intermediate_size % MOONEP_WEIGHT_TILE: + raise ValueError( + "MoonEP weight prefetch requires hidden_size and intermediate_size " + f"to be multiples of {MOONEP_WEIGHT_TILE}; got H={hidden_size}, " + f"I={intermediate_size}" + ) + + full_gate_weight = torch.empty( + num_experts + num_prefetch_slots, + intermediate_size, + hidden_size, + dtype=torch.bfloat16, + device=w13_weight.device, + ) + full_up_weight = torch.empty_like(full_gate_weight) + full_down_weight = torch.empty( + num_experts + num_prefetch_slots, + hidden_size, + intermediate_size, + dtype=torch.bfloat16, + device=w2_weight.device, + ) + + full_gate_weight[:num_experts].copy_(w13_weight[:, :intermediate_size, :]) + full_up_weight[:num_experts].copy_(w13_weight[:, intermediate_size:, :]) + full_down_weight[:num_experts].copy_(w2_weight) + full_gate_weight[num_experts:].zero_() + full_up_weight[num_experts:].zero_() + full_down_weight[num_experts:].zero_() + + return MoonEPExpertWeightLayout( + full_gate_weight=full_gate_weight.contiguous(), + full_up_weight=full_up_weight.contiguous(), + full_down_weight=full_down_weight.contiguous(), + num_prefetch_slots=num_prefetch_slots, + ) + + +def gather_moonep_weight_layout( + w13_local: torch.Tensor, + w2_local: torch.Tensor, + num_global_experts: int, + num_prefetch_slots: int, +) -> MoonEPExpertWeightLayout: + """Build the replicated ``[E+B, ...]`` layout from this rank's local experts. + + PoC bridge: each EP rank loads only its own experts (linear placement), + so all-gather them once at load time into global expert order on every + rank. Production MoonEP instead maps rows ``[0, E)`` onto each home + rank's parameter memory via symmetric memory (RFC #52095 item 5). + """ + from vllm.distributed import get_ep_group + + ep_group = get_ep_group() + ep_size = ep_group.world_size + if ep_size == 1: + return make_moonep_weight_layout(w13_local, w2_local, num_prefetch_slots) + if w13_local.size(0) * ep_size != num_global_experts: + raise NotImplementedError( + "MoonEP PoC requires num_experts to be evenly divisible across EP " + f"ranks: {num_global_experts} experts, {ep_size} ranks" + ) + w13_global = ep_group.all_gather(w13_local.contiguous(), dim=0) + w2_global = ep_group.all_gather(w2_local.contiguous(), dim=0) + return make_moonep_weight_layout(w13_global, w2_global, num_prefetch_slots) + + +MOONEP_MIN_DISPATCH_TOKENS = 128 + + +class MoonEPBufferPool: + """Lazily creates ``moonep.Buffer`` instances at power-of-two capacities. + + ``Buffer`` fixes its token capacity ``S`` at construction, so a single + full-capacity buffer would pad every dispatch to + ``max_num_batched_tokens``. The pool instead serves the smallest + power-of-two capacity that fits the step's token count, so decode-sized + batches dispatch a few hundred slots rather than the full capacity. + + Selection must be identical on every EP rank (dispatch is collective and + ``Buffer`` construction is itself a collective): callers derive the token + count from ``dp_metadata`` (the max across DP ranks) so all ranks create + and pick the same buffer at the same step. + """ + + def __init__(self, buffer_kwargs: dict[str, Any], max_tokens_per_rank: int): + self._kwargs = {k: v for k, v in buffer_kwargs.items() if k != "S"} + self.max_tokens_per_rank = max_tokens_per_rank + self._buffers: dict[int, Any] = {} + + def capacity_for(self, num_tokens: int) -> int: + if num_tokens >= self.max_tokens_per_rank: + return self.max_tokens_per_rank + cap = 1 << max(num_tokens - 1, 0).bit_length() + return min(max(cap, MOONEP_MIN_DISPATCH_TOKENS), self.max_tokens_per_rank) + + def get(self, num_tokens: int) -> tuple[int, Any]: + """Return ``(capacity, buffer)`` for the given step token count.""" + from moonep import Buffer # type: ignore[import-not-found] + + capacity = self.capacity_for(num_tokens) + buffer = self._buffers.get(capacity) + if buffer is None: + buffer = Buffer(S=capacity, **self._kwargs) + self._buffers[capacity] = buffer + return capacity, buffer + + def destroy(self) -> None: + for buffer in self._buffers.values(): + buffer.destroy() + self._buffers.clear() + + +class MoonEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): + """Prepare/Finalize using MoonEP balanced dispatch/combine. + + ``prepare`` pads the batch to the selected buffer's static token + capacity, dispatches, runs ``prefetch_weight`` for the planned redundant + experts, and stashes the ``plan`` for ``finalize`` (the same pattern + DeepEP-HT uses for its handle). Downstream expert compute must consume + the expert-grouped ``[NvS, H]`` layout via ``cu_seqlens``. + """ + + def __init__( + self, + buffer_pool: "MoonEPBufferPool", + max_tokens_per_rank: int, + num_dispatchers: int, + num_global_experts: int, + weight_layout: MoonEPExpertWeightLayout | None = None, + ): + super().__init__() + self.buffer_pool = buffer_pool + self.max_tokens_per_rank = max_tokens_per_rank + self.num_dispatchers_ = num_dispatchers + self.num_global_experts = num_global_experts + self.weight_layout = weight_layout + self._fused_experts: Any = None + + # dispatch state consumed by finalize (and the expert runner) + self._plan: Any = None + self._cu_seqlens: torch.Tensor | None = None + self._num_tokens: int = 0 + self._buffer: Any = None # the pool buffer used by the current step + + def post_init_setup(self, fused_experts: mk.FusedMoEExperts) -> None: + # The [E+B] weight layout is attached to the experts by their + # process_weights_after_loading hook (after this runs), so keep the + # reference and resolve the layout lazily in prepare(). + self._fused_experts = fused_experts + + def _resolve_weight_layout(self) -> MoonEPExpertWeightLayout: + if self.weight_layout is None and self._fused_experts is not None: + layout = getattr(self._fused_experts, "weight_layout", None) + if isinstance(layout, MoonEPExpertWeightLayout): + self.weight_layout = layout + # Redundant experts' weights must be in rows [E, E+B) before the + # expert compute reads them; skipping prefetch silently corrupts + # output. + assert self.weight_layout is not None, ( + "MoonEPPrepareAndFinalize: weight layout not available (the " + "experts' process_weights_after_loading has not run)" + ) + return self.weight_layout + + @property + def num_dispatched_slots(self) -> int: + """``NvS`` of the buffer selected by the current step's prepare().""" + assert self._buffer is not None, "prepare() has not been called" + return int(self._buffer._ctx["NvS"]) + + def num_dispatchers(self) -> int: + return self.num_dispatchers_ + + def output_is_reduced(self) -> bool: + # combine returns the fully weighted+reduced token-major output + return True + + @property + def activation_format(self) -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + def max_num_tokens_per_rank(self) -> int | None: + return self.max_tokens_per_rank + + def topk_indices_dtype(self) -> torch.dtype | None: + return torch.int32 + + def supports_async(self) -> bool: + return False + + @property + def cu_seqlens(self) -> torch.Tensor: + assert self._cu_seqlens is not None, "prepare() has not been called" + return self._cu_seqlens + + @property + def plan(self) -> Any: + assert self._plan is not None, "prepare() has not been called" + return self._plan + + def _pad_to_capacity( + self, + a1: torch.Tensor, + topk_ids: torch.Tensor, + topk_weights: torch.Tensor, + capacity: int, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]: + num_tokens = a1.shape[0] + if num_tokens > capacity: + raise ValueError( + f"MoonEP static buffer capacity exceeded: {num_tokens} tokens " + f"> S={capacity}." + ) + a1 = a1.contiguous() + topk_ids = topk_ids.to(dtype=torch.int32) + topk_weights = topk_weights.to(dtype=torch.float32) + # MoonEP's planner has no sentinel for invalid expert ids, so map + # negative ids (padded/invalid slots from the engine) to expert 0 + # with zero route weight — the same convention as the capacity + # padding rows below. Wasted compute, never wrong output. + invalid = topk_ids < 0 + topk_ids = torch.where(invalid, torch.zeros_like(topk_ids), topk_ids) + topk_weights = torch.where( + invalid, torch.zeros_like(topk_weights), topk_weights + ).contiguous() + topk_ids = topk_ids.contiguous() + if num_tokens == capacity: + return a1, topk_ids, topk_weights, num_tokens + pad = capacity - num_tokens + return ( + F.pad(a1, (0, 0, 0, pad)), + F.pad(topk_ids, (0, 0, 0, pad)), + F.pad(topk_weights, (0, 0, 0, pad)), + num_tokens, + ) + + def prepare( + self, + a1: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + num_experts: int, + expert_map: torch.Tensor | None, + apply_router_weight_on_input: bool, + quant_config: FusedMoEQuantConfig, + defer_input_quant: bool = False, + ) -> mk.PrepareResultType: + if a1.dtype != torch.bfloat16: + raise NotImplementedError("MoonEP PoC supports BF16 hidden states only.") + if quant_config.quant_dtype is not None: + raise NotImplementedError("MoonEP PoC does not support quantized dispatch.") + assert num_experts == self.num_global_experts + assert self._plan is None, ( + "MoonEPPrepareAndFinalize.prepare() called again before finalize()" + ) + if apply_router_weight_on_input: + assert topk_ids.size(1) == 1, ( + "apply_router_weight_on_input requires top-1 routing" + ) + a1 = a1 * topk_weights.to(a1.dtype) + + # Buffer choice must be rank-symmetric: use the max token count + # across DP ranks (every rank sees the same dp_metadata), falling + # back to the local count outside a forward context (tests). + num_tokens = a1.shape[0] + try: + dp_meta = get_forward_context().dp_metadata + except AssertionError: + dp_meta = None + if dp_meta is not None: + num_tokens = int(dp_meta.num_tokens_across_dp_cpu.max()) + capacity, self._buffer = self.buffer_pool.get(num_tokens) + + a1, topk_ids, topk_weights, self._num_tokens = self._pad_to_capacity( + a1, topk_ids, topk_weights, capacity + ) + tokens_per_expert = torch.bincount( + topk_ids.reshape(-1).to(dtype=torch.int64), + minlength=num_experts, + ).to(dtype=torch.int32) + + hidden_nvsh, route_weights_nvs, cu_seqlens, plan = self._buffer.dispatch( + a1, + topk_weights, + topk_ids, + tokens_per_expert, + ) + self._plan = plan + self._cu_seqlens = cu_seqlens + + weight_layout = self._resolve_weight_layout() + self._buffer.prefetch_weight( + plan=plan, + full_gate_weight=weight_layout.full_gate_weight, + full_up_weight=weight_layout.full_up_weight, + full_down_weight=weight_layout.full_down_weight, + ) + + # Segment sizes per [E+B] weight row. NOTE: this is per *row*, not + # per local expert, and NvS rows are expert-grouped rather than + # token-major — only MoonEP-aware expert implementations can consume + # this activation layout. + expert_num_tokens = torch.diff(cu_seqlens, prepend=cu_seqlens.new_zeros(1)) + expert_tokens_meta = mk.ExpertTokensMetadata( + expert_num_tokens=expert_num_tokens, + expert_num_tokens_cpu=None, + ) + + # MoonEP has no post-dispatch per-token topk ids/weights; route + # weights come back in NvS order. + return hidden_nvsh, None, expert_tokens_meta, None, route_weights_nvs + + def finalize( + self, + output: torch.Tensor, + fused_expert_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + apply_router_weight_on_input: bool, + weight_and_reduce_impl: mk.TopKWeightAndReduce, + ) -> None: + # Route weights are applied in the expert compute and MoonEP's + # combine performs the K-sum, so any weight/reduce work in finalize + # means MoonEP was paired with the wrong kind of experts. + assert isinstance(weight_and_reduce_impl, TopKWeightAndReduceNoOP), ( + "MoonEP requires TopKWeightAndReduceNoOP, got " + f"{type(weight_and_reduce_impl).__name__}" + ) + combined, _, _ = self._buffer.combine( + plan=self.plan, + hidden_nvsh=fused_expert_output, + route_weights_nvs=None, + ) + output.copy_(combined[: self._num_tokens]) + self._plan = None + self._cu_seqlens = None diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index e0835f538c..17481d4a91 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -49,13 +49,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # --8<-- [end:unquantized_fused_moe] - supports_pre_processed_weights = True - def __init__(self, moe: FusedMoEConfig): super().__init__(moe) self.unquantized_backend, self.experts_cls = select_unquantized_moe_backend( moe_config=self.moe, ) + # MoonEP replaces the named parameters with its [E+B] gate/down + # split and keeps the up projection outside the parameter registry, + # so a pre-processed (e.g. ipc_cache) load cannot reconstruct it. + self.supports_pre_processed_weights = ( + self.unquantized_backend != UnquantizedMoeBackend.MOONEP + ) @property def supports_eplb(self) -> bool: @@ -171,6 +175,7 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): moe_config=layer.moe_config, w13_weight=w13, w2_weight=w2, + layer=layer, ) # `moe_kernel` is initialized to None in FusedMoEMethodBase.__init__; # On the first call we replace the parameter normally. On subsequent @@ -191,14 +196,13 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): # does not need to be re-built. self._init_moe_kernel(layer) - if self.unquantized_backend == UnquantizedMoeBackend.CPU: - # The CPU experts need the layer itself for the setup that - # convert_to_unquantized_kernel_format cannot express, since - # it only sees the two weight tensors: padding and prepacking - # into the grouped-gemm layout (bias included), and capturing - # the router config that monolithic apply() cannot carry. - assert self.moe_kernel is not None - self.moe_kernel.fused_experts.process_weights_after_loading(layer) + # No-op by default. Experts that need the layer itself for setup + # convert_to_unquantized_kernel_format cannot express override + # it: CPU prepacks into its grouped-gemm layout and captures the + # router config; MoonEP picks up the [E+B] weight layout for + # prefetch and the up projection. + assert self.moe_kernel is not None + self.moe_kernel.fused_experts.process_weights_after_loading(layer) def _init_moe_kernel(self, layer: "RoutedExperts") -> None: """Build the MoE kernel from the layer's current (shuffled) weights.""" diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 4e90367a3e..aaacccc7cb 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -512,6 +512,34 @@ def has_nixl_ep() -> bool: return _has_module("nixl_ep") +def has_moonep() -> bool: + """Whether the optional `moonep` package is available.""" + return _has_module("moonep") + + +def check_moonep_system_support() -> None: + """Raise if the current device cannot run MoonEP. + + MoonEP's symmetric-memory buffers require CUDA VMM plus NVSwitch multicast + (SHARP) on every EP rank; NVLink-only topologies without NVSwitch (e.g. + 4x H100 NV6) fail deep inside ``moonep.Buffer`` otherwise. + """ + if not has_moonep(): + raise RuntimeError( + "MoonEP not available. Install it from " + "https://github.com/MoonshotAI/MoonEP." + ) + from moonep._C import ( # type: ignore[import-not-found] + nvl_multicast_supported, + ) + + if not nvl_multicast_supported(): + raise RuntimeError( + "MoonEP requires NVSwitch multicast (SHARP) support on the " + "current device; it is not supported on this GPU/topology." + ) + + def is_numba_available() -> bool: """Whether the optional `numba` package is available.""" return _has_module("numba")