[ROCm][DSv4] AITER MegaMoE Fixed-Slot Dispatch For DeepSeek-V4 (#61262)
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
This commit is contained in:
@@ -110,6 +110,31 @@ def test_experts_load_into_local_slots(monkeypatch):
|
||||
assert torch.all(w2 == 10 * expert + 3)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"tokens_across_dp, expected",
|
||||
[([4, 128], "decode"), ([4, 129], "prefill")],
|
||||
)
|
||||
def test_runtime_selected_by_max_tokens_across_dp(
|
||||
monkeypatch, tokens_across_dp, expected
|
||||
):
|
||||
experts = _make_experts(monkeypatch)
|
||||
experts._runtimes = [(128, "decode"), (4096, "prefill")]
|
||||
monkeypatch.setattr(mega_moe_module, "is_forward_context_available", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
mega_moe_module,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(
|
||||
dp_metadata=SimpleNamespace(
|
||||
num_tokens_across_dp_cpu=torch.tensor(tokens_across_dp)
|
||||
)
|
||||
),
|
||||
)
|
||||
# Every rank must take the same runtime, so a small local batch still gets
|
||||
# the prefill runtime when another DP rank is over the decode capacity.
|
||||
assert experts.select_runtime(4) == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
def test_shared_expert_route_appended(monkeypatch):
|
||||
experts = _make_experts(
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch.nn as nn
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
@@ -20,7 +21,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.math_utils import next_power_of_2, prev_power_of_2
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -108,7 +109,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
|
||||
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
|
||||
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
|
||||
self._runtime: Any = None
|
||||
# (max_tok_per_rank, runtime), ascending by capacity.
|
||||
self._runtimes: list[tuple[int, Any]] = []
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
@@ -150,7 +152,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
weight_scale.data[slot].copy_(scale.view(torch.uint8))
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._runtime is not None:
|
||||
if self._runtimes:
|
||||
return
|
||||
n = self.num_local_experts
|
||||
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
@@ -165,6 +167,35 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
|
||||
)
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
FIXED_SLOT_MAX_MTPR,
|
||||
)
|
||||
|
||||
# MegaMoEV2 takes fixed-slot dispatch up to FIXED_SLOT_MAX_MTPR (255 by
|
||||
# default, raised via AITER_MEGA_FIXED_SLOT_MAX_MTPR); max_tok_per_rank
|
||||
# must also be a power of two.
|
||||
fixed_slot_max_tokens = prev_power_of_2(FIXED_SLOT_MAX_MTPR)
|
||||
capacities = [self.max_num_tokens]
|
||||
if self.max_num_tokens > fixed_slot_max_tokens and self._uses_fixed_slot(
|
||||
fixed_slot_max_tokens
|
||||
):
|
||||
capacities.insert(0, fixed_slot_max_tokens)
|
||||
self._runtimes = [(c, self.get_runtime(c)) for c in capacities]
|
||||
|
||||
def _uses_fixed_slot(self, max_tok_per_rank: int) -> bool:
|
||||
from aiter.ops.flydsl.kernels.mega_moe.mega_moe_config import (
|
||||
build_mega_moe_bundle_plan,
|
||||
)
|
||||
|
||||
return build_mega_moe_bundle_plan(
|
||||
max_tok_per_rank,
|
||||
experts_per_rank=self.num_local_experts,
|
||||
model_dim=self.hidden_size,
|
||||
inter_dim=self.intermediate_size,
|
||||
world_size=self.ep_size,
|
||||
).fixed_slot_dispatch
|
||||
|
||||
def get_runtime(self, max_tok_per_rank: int) -> Any:
|
||||
key = (
|
||||
id(get_ep_group().cpu_group),
|
||||
torch.accelerator.current_device_index(),
|
||||
@@ -172,7 +203,7 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.max_num_tokens,
|
||||
max_tok_per_rank,
|
||||
self.swiglu_limit,
|
||||
)
|
||||
runtime = self._runtime_cache.get(key)
|
||||
@@ -191,11 +222,26 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
w1_scale=self.w13_weight_scale,
|
||||
w2=self.w2_weight,
|
||||
w2_scale=self.w2_weight_scale,
|
||||
max_tok_per_rank=self.max_num_tokens,
|
||||
max_tok_per_rank=max_tok_per_rank,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
)
|
||||
self._runtime_cache[key] = runtime
|
||||
self._runtime = runtime
|
||||
return runtime
|
||||
|
||||
def select_runtime(self, num_tokens: int) -> Any:
|
||||
# Dispatch is collective, so every EP rank must pick the same runtime.
|
||||
if is_forward_context_available():
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is not None:
|
||||
num_tokens = max(
|
||||
num_tokens, int(dp_metadata.num_tokens_across_dp_cpu.max())
|
||||
)
|
||||
for max_tok_per_rank, runtime in self._runtimes:
|
||||
if num_tokens <= max_tok_per_rank:
|
||||
return runtime
|
||||
raise ValueError(
|
||||
f"{num_tokens} tokens exceeds MegaMoE capacity {self.max_num_tokens}"
|
||||
)
|
||||
|
||||
def append_shared_expert(
|
||||
self, topk_weights: torch.Tensor, topk_ids: torch.Tensor
|
||||
@@ -230,8 +276,8 @@ class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
runtime = self._runtime
|
||||
assert runtime is not None
|
||||
assert self._runtimes
|
||||
runtime = self.select_runtime(hidden_states.shape[0])
|
||||
if self.fuse_shared_expert:
|
||||
topk_weights, topk_ids = self.append_shared_expert(topk_weights, topk_ids)
|
||||
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
||||
|
||||
@@ -19,6 +19,11 @@ def next_power_of_2(n: int) -> int:
|
||||
return 1 if n < 1 else 1 << (n - 1).bit_length()
|
||||
|
||||
|
||||
def prev_power_of_2(n: int) -> int:
|
||||
"""The previous power of 2 (inclusive)."""
|
||||
return 0 if n < 1 else 1 << (n.bit_length() - 1)
|
||||
|
||||
|
||||
def round_up(x: int, y: int) -> int:
|
||||
"""Round up x to the nearest multiple of y."""
|
||||
return ((x + y - 1) // y) * y
|
||||
|
||||
Reference in New Issue
Block a user