diff --git a/tests/models/test_deepseek_v4_aiter_mega_moe.py b/tests/models/test_deepseek_v4_aiter_mega_moe.py index 19b3697811..1a53476694 100644 --- a/tests/models/test_deepseek_v4_aiter_mega_moe.py +++ b/tests/models/test_deepseek_v4_aiter_mega_moe.py @@ -110,6 +110,31 @@ def test_experts_load_into_local_slots(monkeypatch): assert torch.all(w2 == 10 * expert + 3) +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "tokens_across_dp, expected", + [([4, 128], "decode"), ([4, 129], "prefill")], +) +def test_runtime_selected_by_max_tokens_across_dp( + monkeypatch, tokens_across_dp, expected +): + experts = _make_experts(monkeypatch) + experts._runtimes = [(128, "decode"), (4096, "prefill")] + monkeypatch.setattr(mega_moe_module, "is_forward_context_available", lambda: True) + monkeypatch.setattr( + mega_moe_module, + "get_forward_context", + lambda: SimpleNamespace( + dp_metadata=SimpleNamespace( + num_tokens_across_dp_cpu=torch.tensor(tokens_across_dp) + ) + ), + ) + # Every rank must take the same runtime, so a small local batch still gets + # the prefill runtime when another DP rank is over the decode capacity. + assert experts.select_runtime(4) == expected + + @pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm") def test_shared_expert_route_appended(monkeypatch): experts = _make_experts( diff --git a/vllm/models/deepseek_v4/amd/mega_moe.py b/vllm/models/deepseek_v4/amd/mega_moe.py index 312c089e5a..336374f77e 100644 --- a/vllm/models/deepseek_v4/amd/mega_moe.py +++ b/vllm/models/deepseek_v4/amd/mega_moe.py @@ -10,6 +10,7 @@ import torch.nn as nn from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig from vllm.distributed import get_ep_group +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.model_executor.layers.quantization.utils.fp8_utils import ( get_fp8_block_weight_scale, ) @@ -20,7 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.model_executor.utils import set_weight_attrs from vllm.triton_utils import tl, triton -from vllm.utils.math_utils import next_power_of_2 +from vllm.utils.math_utils import next_power_of_2, prev_power_of_2 @triton.jit @@ -108,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32) self.w2_weight = make_param(hidden_size, intermediate_size // 2) self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32) - self._runtime: Any = None + # (max_tok_per_rank, runtime), ascending by capacity. + self._runtimes: list[tuple[int, Any]] = [] def weight_loader( self, @@ -150,7 +152,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): weight_scale.data[slot].copy_(scale.view(torch.uint8)) def finalize_weights(self) -> None: - if self._runtime is not None: + if self._runtimes: return n = self.num_local_experts self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4( @@ -165,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4( self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False ) + from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import ( + FIXED_SLOT_MAX_MTPR, + ) + + # MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by + # default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank + # must also be a power of two. + fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR) + capacities = [self.max_num_tokens] + if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot( + fixed_slot_max_tokens + ): + capacities.insert(0, fixed_slot_max_tokens) + self._runtimes = [(c, self.get_runtime(c)) for c in capacities] + + def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool: + from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import ( + build_mega_moe_bundle_plan, + ) + + return build_mega_moe_bundle_plan( + max_tok_per_rank, + experts_per_rank=self.num_local_experts, + model_dim=self.hidden_size, + inter_dim=self.intermediate_size, + world_size=self.ep_size, + ).fixed_slot_dispatch + + def get_runtime(self, max_tok_per_rank: int) -> Any: key = ( id(get_ep_group().cpu_group), torch.accelerator.current_device_index(), @@ -172,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): self.top_k, self.hidden_size, self.intermediate_size, - self.max_num_tokens, + max_tok_per_rank, self.swiglu_limit, ) runtime = self._runtime_cache.get(key) @@ -191,11 +222,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): w1_scale=self.w13_weight_scale, w2=self.w2_weight, w2_scale=self.w2_weight_scale, - max_tok_per_rank=self.max_num_tokens, + max_tok_per_rank=max_tok_per_rank, swiglu_limit=self.swiglu_limit, ) self._runtime_cache[key] = runtime - self._runtime = runtime + return runtime + + def select_runtime(self, num_tokens: int) -> Any: + # Dispatch is collective, so every EP rank must pick the same runtime. + if is_forward_context_available(): + dp_metadata = get_forward_context().dp_metadata + if dp_metadata is not None: + num_tokens = max( + num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max()) + ) + for max_tok_per_rank, runtime in self._runtimes: + if num_tokens <= max_tok_per_rank: + return runtime + raise ValueError( + f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}" + ) def append_shared_expert( self, topk_weights: torch.Tensor, topk_ids: torch.Tensor @@ -230,8 +276,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module): topk_weights: torch.Tensor, topk_ids: torch.Tensor, ) -> torch.Tensor: - runtime = self._runtime - assert runtime is not None + assert self._runtimes + runtime = self.select_runtime(hidden_states.shape[0]) if self.fuse_shared_expert: topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids) # MegaMoEV2 reads these per launch; rebind to this layer's experts. diff --git a/vllm/utils/math_utils.py b/vllm/utils/math_utils.py index 9854249af2..d07fe7cf74 100644 --- a/vllm/utils/math_utils.py +++ b/vllm/utils/math_utils.py @@ -19,6 +19,11 @@ def next_power_of_2(n: int) -> int: return 1 if n < 1 else 1 << (n - 1).bit_length() +def prev_power_of_2(n: int) -> int: + """The previous power of 2 (inclusive).""" + return 0 if n < 1 else 1 << (n.bit_length() - 1) + + def round_up(x: int, y: int) -> int: """Round up x to the nearest multiple of y.""" return ((x + y - 1) // y) * y