# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4.""" from typing import Any import torch import torch.nn as nn from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.distributed import get_ep_group from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.quantization.utils.fp8_utils import ( get_fp8_block_weight_scale, ) from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mxfp4_quantize from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, scaled_dequantize, ) from vllm.model_executor.utils import set_weight_attrs from vllm.triton_utils import tl, triton from vllm.utils.math_utils import next_power_of_2, prev_power_of_2 @triton.jit def _append_shared_expert_kernel( topk_ids_ptr, topk_weights_ptr, out_ids_ptr, out_weights_ptr, num_tokens, shared_expert_id, TOPK: tl.constexpr, ROUTED_PER_RANK: tl.constexpr, BLOCK_M: tl.constexpr, ): rows = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M) mask = rows < num_tokens for k in tl.static_range(TOPK): e = tl.load(topk_ids_ptr + rows * TOPK + k, mask=mask, other=0) w = tl.load(topk_weights_ptr + rows * TOPK + k, mask=mask, other=0.0) e = tl.where(e >= 0, e + e // ROUTED_PER_RANK, e) tl.store(out_ids_ptr + rows * (TOPK + 1) + k, e, mask=mask) tl.store(out_weights_ptr + rows * (TOPK + 1) + k, w, mask=mask) tl.store( out_ids_ptr + rows * (TOPK + 1) + TOPK, tl.zeros_like(rows) + shared_expert_id, mask=mask, ) tl.store( out_weights_ptr + rows * (TOPK + 1) + TOPK, tl.full([BLOCK_M], 1.0, tl.float32), mask=mask, ) class DeepseekV4AiterMegaMoEExperts(nn.Module): # MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one # instance and passes its own weights per call. _runtime_cache: dict[tuple, Any] = {} def __init__( self, vllm_config: VllmConfig, *, num_experts: int, top_k: int, hidden_size: int, intermediate_size: int, swiglu_limit: float | None, fuse_shared_expert: bool = False, prefix: str = "", ): super().__init__() ep_group = get_ep_group() self.ep_rank = ep_group.rank_in_group self.ep_size = ep_group.world_size self.num_routed_local_experts = num_experts // self.ep_size self.experts_start_idx = self.ep_rank * self.num_routed_local_experts # A fused shared expert is one extra local expert per rank and one # extra route per token, to the token's own rank. self.fuse_shared_expert = fuse_shared_expert self.num_local_experts = self.num_routed_local_experts + fuse_shared_expert self.num_experts = self.num_local_experts * self.ep_size self.routed_top_k = top_k self.top_k = top_k + fuse_shared_expert self.shared_expert_id = ( self.ep_rank * self.num_local_experts + self.num_routed_local_experts ) self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.swiglu_limit = swiglu_limit or 0.0 self.max_num_tokens = next_power_of_2( vllm_config.scheduler_config.max_num_batched_tokens ) self.prefix = prefix def make_param(rows: int, cols: int) -> nn.Parameter: param = nn.Parameter( torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8), requires_grad=False, ) set_weight_attrs(param, {"weight_loader": self.weight_loader}) return param self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2) self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32) self.w2_weight = make_param(hidden_size, intermediate_size // 2) self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32) # (max_tok_per_rank, runtime), ascending by capacity. self._runtimes: list[tuple[int, Any]] = [] def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str, shard_id: str, expert_id: int, return_success: bool = False, ) -> bool | None: local_expert_id = expert_id - self.experts_start_idx if not 0 <= local_expert_id < self.num_routed_local_experts: return False if return_success else None expert_data = param.data[local_expert_id] if shard_id in ("w1", "w3"): shard_offset = 0 if shard_id == "w1" else self.intermediate_size expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size) elif shard_id != "w2": raise ValueError(f"Unsupported expert shard id: {shard_id}") expert_data.copy_(loaded_weight.view(torch.uint8)) return True if return_success else None def load_shared_expert(self, shared_experts: nn.Module) -> None: slot = self.num_routed_local_experts for linear, weight, weight_scale in ( (shared_experts.gate_up_proj, self.w13_weight, self.w13_weight_scale), (shared_experts.down_proj, self.w2_weight, self.w2_weight_scale), ): block_scale = get_fp8_block_weight_scale(linear) assert block_scale is not None, "shared expert must be block-FP8" group = GroupShape( linear.weight.shape[0] // block_scale.shape[0], linear.weight.shape[1] // block_scale.shape[1], ) packed, scale = mxfp4_quantize( scaled_dequantize(linear.weight, block_scale, group, torch.bfloat16) ) weight.data[slot].copy_(packed.view(torch.uint8)) weight_scale.data[slot].copy_(scale.view(torch.uint8)) def finalize_weights(self) -> None: if self._runtimes: return n = self.num_local_experts self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4( self.w13_weight.data, 16, True ) self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4( self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True ) self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4( self.w2_weight.data, 16, False ) self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4( self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False ) from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import ( FIXED_SLOT_MAX_MTPR, ) # MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by # default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank # must also be a power of two. fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR) capacities = [self.max_num_tokens] if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot( fixed_slot_max_tokens ): capacities.insert(0, fixed_slot_max_tokens) self._runtimes = [(c, self.get_runtime(c)) for c in capacities] def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool: from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import ( build_mega_moe_bundle_plan, ) return build_mega_moe_bundle_plan( max_tok_per_rank, experts_per_rank=self.num_local_experts, model_dim=self.hidden_size, inter_dim=self.intermediate_size, world_size=self.ep_size, ).fixed_slot_dispatch def get_runtime(self, max_tok_per_rank: int) -> Any: key = ( id(get_ep_group().cpu_group), torch.accelerator.current_device_index(), self.num_experts, self.top_k, self.hidden_size, self.intermediate_size, max_tok_per_rank, self.swiglu_limit, ) runtime = self._runtime_cache.get(key) if runtime is None: from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2 runtime = MegaMoEV2( rank=self.ep_rank, world_size=self.ep_size, model_dim=self.hidden_size, inter_dim=self.intermediate_size, experts=self.num_experts, topk=self.top_k, quant="a8w4", w1=self.w13_weight, w1_scale=self.w13_weight_scale, w2=self.w2_weight, w2_scale=self.w2_weight_scale, max_tok_per_rank=max_tok_per_rank, swiglu_limit=self.swiglu_limit, ) self._runtime_cache[key] = runtime return runtime def select_runtime(self, num_tokens: int) -> Any: # Dispatch is collective, so every EP rank must pick the same runtime. if is_forward_context_available(): dp_metadata = get_forward_context().dp_metadata if dp_metadata is not None: num_tokens = max( num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max()) ) for max_tok_per_rank, runtime in self._runtimes: if num_tokens <= max_tok_per_rank: return runtime raise ValueError( f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}" ) def append_shared_expert( self, topk_weights: torch.Tensor, topk_ids: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: """Remap routed ids to the per-rank (routed + shared) layout and append a weight-1.0 route to this rank's shared slot.""" num_tokens = topk_ids.shape[0] out_ids = torch.empty( (num_tokens, self.top_k), dtype=topk_ids.dtype, device=topk_ids.device ) out_weights = torch.empty( (num_tokens, self.top_k), dtype=torch.float32, device=topk_ids.device ) if num_tokens: block_m = 64 _append_shared_expert_kernel[(triton.cdiv(num_tokens, block_m),)]( topk_ids.contiguous(), topk_weights.to(torch.float32).contiguous(), out_ids, out_weights, num_tokens, self.shared_expert_id, TOPK=self.routed_top_k, ROUTED_PER_RANK=self.num_routed_local_experts, BLOCK_M=block_m, ) return out_weights, out_ids def forward( self, hidden_states: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, ) -> torch.Tensor: assert self._runtimes runtime = self.select_runtime(hidden_states.shape[0]) if self.fuse_shared_expert: topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids) # MegaMoEV2 reads these per launch; rebind to this layer's experts. runtime._s1_w1 = self.w13_weight runtime._s1_w1_scale = self.w13_weight_scale runtime.w2 = self.w2_weight runtime.w2_scale = self.w2_weight_scale return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids) DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined] def finalize_mega_moe_weights(model: nn.Module) -> None: for module in list(model.modules()): experts = getattr(module, "experts", None) if isinstance(experts, DeepseekV4AiterMegaMoEExperts): if experts.fuse_shared_expert: experts.load_shared_expert(module.shared_experts) module.shared_experts = None experts.finalize_weights()