[ROCm][DSv4] AITER MegaMoE Fixed-Slot Dispatch For DeepSeek-V4 (#61262)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
Micah Williamson
2026-10-15 19:10:59 +08:00
committed by GitHub
parent 8b27d27402
commit e8e1e96196
3 changed files with 84 additions and 8 deletions
+54 -8
View File
@@ -10,6 +10,7 @@ import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
get_fp8_block_weight_scale,
)
@@ -20,7 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
@triton.jit
@@ -108,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
self._runtime: Any = None
# (max_tok_per_rank, runtime), ascending by capacity.
self._runtimes: list[tuple[int, Any]] = []
def weight_loader(
self,
@@ -150,7 +152,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
weight_scale.data[slot].copy_(scale.view(torch.uint8))
def finalize_weights(self) -> None:
if self._runtime is not None:
if self._runtimes:
return
n = self.num_local_experts
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
@@ -165,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
)
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
FIXED_SLOT_MAX_MTPR,
)
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
# must also be a power of two.
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
capacities = [self.max_num_tokens]
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
fixed_slot_max_tokens
):
capacities.insert(0, fixed_slot_max_tokens)
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
build_mega_moe_bundle_plan,
)
return build_mega_moe_bundle_plan(
max_tok_per_rank,
experts_per_rank=self.num_local_experts,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
world_size=self.ep_size,
).fixed_slot_dispatch
def get_runtime(self, max_tok_per_rank: int) -> Any:
key = (
id(get_ep_group().cpu_group),
torch.accelerator.current_device_index(),
@@ -172,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
self.top_k,
self.hidden_size,
self.intermediate_size,
self.max_num_tokens,
max_tok_per_rank,
self.swiglu_limit,
)
runtime = self._runtime_cache.get(key)
@@ -191,11 +222,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
w1_scale=self.w13_weight_scale,
w2=self.w2_weight,
w2_scale=self.w2_weight_scale,
max_tok_per_rank=self.max_num_tokens,
max_tok_per_rank=max_tok_per_rank,
swiglu_limit=self.swiglu_limit,
)
self._runtime_cache[key] = runtime
self._runtime = runtime
return runtime
def select_runtime(self, num_tokens: int) -> Any:
# Dispatch is collective, so every EP rank must pick the same runtime.
if is_forward_context_available():
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is not None:
num_tokens = max(
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
)
for max_tok_per_rank, runtime in self._runtimes:
if num_tokens <= max_tok_per_rank:
return runtime
raise ValueError(
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
)
def append_shared_expert(
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
@@ -230,8 +276,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
runtime = self._runtime
assert runtime is not None
assert self._runtimes
runtime = self.select_runtime(hidden_states.shape[0])
if self.fuse_shared_expert:
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
# MegaMoEV2 reads these per launch; rebind to this layer's experts.