[ROCm][DSv4] AITER MegaMoE Fixed-Slot Dispatch For DeepSeek-V4 (#61262)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
Micah Williamson
2026-10-15 19:10:59 +08:00
committed by GitHub
parent 8b27d27402
commit e8e1e96196
3 changed files with 84 additions and 8 deletions
@@ -110,6 +110,31 @@ def test_experts_load_into_local_slots(monkeypatch):
assert torch.all(w2 == 10 * expert + 3)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"tokens_across_dp, expected",
[([4, 128], "decode"), ([4, 129], "prefill")],
)
def test_runtime_selected_by_max_tokens_across_dp(
monkeypatch, tokens_across_dp, expected
):
experts = _make_experts(monkeypatch)
experts._runtimes = [(128, "decode"), (4096, "prefill")]
monkeypatch.setattr(mega_moe_module, "is_forward_context_available", lambda: True)
monkeypatch.setattr(
mega_moe_module,
"get_forward_context",
lambda: SimpleNamespace(
dp_metadata=SimpleNamespace(
num_tokens_across_dp_cpu=torch.tensor(tokens_across_dp)
)
),
)
# Every rank must take the same runtime, so a small local batch still gets
# the prefill runtime when another DP rank is over the decode capacity.
assert experts.select_runtime(4) == expected
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
def test_shared_expert_route_appended(monkeypatch):
experts = _make_experts(
+54 -8
View File
@@ -10,6 +10,7 @@ import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
@@ -20,7 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
@triton.jit
@@ -108,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
self._runtime: Any = None
# (max_tok_per_rank, runtime), ascending by capacity.
self._runtimes: list[tuple[int, Any]] = []
def weight_loader(
self,
@@ -150,7 +152,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
weight_scale.data[slot].copy_(scale.view(torch.uint8))
def finalize_weights(self) -> None:
if self._runtime is not None:
if self._runtimes:
return
n = self.num_local_experts
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
@@ -165,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
)
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
FIXED_SLOT_MAX_MTPR,
)
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
# must also be a power of two.
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
capacities = [self.max_num_tokens]
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
fixed_slot_max_tokens
):
capacities.insert(0, fixed_slot_max_tokens)
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
build_mega_moe_bundle_plan,
)
return build_mega_moe_bundle_plan(
max_tok_per_rank,
experts_per_rank=self.num_local_experts,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
world_size=self.ep_size,
).fixed_slot_dispatch
def get_runtime(self, max_tok_per_rank: int) -> Any:
key = (
id(get_ep_group().cpu_group),
torch.accelerator.current_device_index(),
@@ -172,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.top_k,
self.hidden_size,
self.intermediate_size,
self.max_num_tokens,
max_tok_per_rank,
self.swiglu_limit,
)
runtime = self._runtime_cache.get(key)
@@ -191,11 +222,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
w1_scale=self.w13_weight_scale,
w2=self.w2_weight,
w2_scale=self.w2_weight_scale,
max_tok_per_rank=self.max_num_tokens,
max_tok_per_rank=max_tok_per_rank,
swiglu_limit=self.swiglu_limit,
)
self._runtime_cache[key] = runtime
self._runtime = runtime
return runtime
def select_runtime(self, num_tokens: int) -> Any:
# Dispatch is collective, so every EP rank must pick the same runtime.
if is_forward_context_available():
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is not None:
num_tokens = max(
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
)
for max_tok_per_rank, runtime in self._runtimes:
if num_tokens <= max_tok_per_rank:
return runtime
raise ValueError(
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
)
def append_shared_expert(
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
@@ -230,8 +276,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
runtime = self._runtime
assert runtime is not None
assert self._runtimes
runtime = self.select_runtime(hidden_states.shape[0])
if self.fuse_shared_expert:
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
+5
View File
@@ -19,6 +19,11 @@ def next_power_of_2(n: int) -> int:
return 1 if n < 1 else 1 << (n - 1).bit_length()
def prev_power_of_2(n: int) -> int:
"""The previous power of 2 (inclusive)."""
return 0 if n < 1 else 1 << (n.bit_length() - 1)
def round_up(x: int, y: int) -> int:
"""Round up x to the nearest multiple of y."""
return ((x + y - 1) // y) * y