[ROCm][DSv4] AITER MegaMoEV2 Integration For DeepSeek V4 (#59685)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Micah Williamson
2026-10-07 09:35:53 -05:00
committed by GitHub
co-authored by Fangzhou Ai
parent c83935b300
commit f6e3944e31
10 changed files with 454 additions and 6 deletions
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.models.deepseek_v4.amd import mega_moe as mega_moe_module
from vllm.models.deepseek_v4.amd.mega_moe import (
DeepseekV4AiterMegaMoEExperts,
)
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
from vllm.platforms import current_platform
HIDDEN = 64
INTER = 64
def _make_experts(
monkeypatch,
*,
ep_rank=0,
ep_size=1,
num_experts=2,
intermediate_size=INTER,
max_num_batched_tokens=3000,
swiglu_limit=None,
):
monkeypatch.setattr(
mega_moe_module,
"get_ep_group",
lambda: SimpleNamespace(
rank_in_group=ep_rank, world_size=ep_size, cpu_group="ep"
),
)
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens)
)
return DeepseekV4AiterMegaMoEExperts(
vllm_config,
num_experts=num_experts,
top_k=2,
hidden_size=HIDDEN,
intermediate_size=intermediate_size,
swiglu_limit=swiglu_limit,
)
@pytest.mark.cpu_test
def test_experts_load_into_local_slots(monkeypatch):
model = torch.nn.Module()
model.ffn = torch.nn.Module()
model.ffn.experts = _make_experts(monkeypatch, ep_rank=1, ep_size=2, num_experts=4)
params = dict(model.named_parameters())
mapping = fused_moe_make_expert_params_mapping(
model,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=4,
routed_experts_prefix="",
)
shapes = {
("w1", "weight"): (INTER, HIDDEN // 2),
("w3", "weight"): (INTER, HIDDEN // 2),
("w2", "weight"): (HIDDEN, INTER // 2),
("w1", "weight_scale"): (INTER, HIDDEN // 32),
("w3", "weight_scale"): (INTER, HIDDEN // 32),
("w2", "weight_scale"): (HIDDEN, INTER // 32),
}
shard_value = {"w1": 1, "w3": 2, "w2": 3}
loaded = {}
for expert in range(4):
for (shard, suffix), shape in shapes.items():
name = f"ffn.experts.{expert}.{shard}.{suffix}"
weight = torch.full(
shape, 10 * expert + shard_value[shard], dtype=torch.uint8
)
for param_name, weight_name, expert_id, shard_id in mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
param = params[mapped]
if param.weight_loader(
param,
weight,
mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
):
loaded[name] = mapped
break
assert sorted({n.split(".")[2] for n in loaded}) == ["2", "3"]
assert len(loaded) == 2 * len(shapes)
for slot, expert in enumerate((2, 3)):
for suffix in ("weight", "weight_scale"):
w13 = params[f"ffn.experts.w13_{suffix}"][slot]
w2 = params[f"ffn.experts.w2_{suffix}"][slot]
assert torch.all(w13[:INTER] == 10 * expert + 1)
assert torch.all(w13[INTER:] == 10 * expert + 2)
assert torch.all(w2 == 10 * expert + 3)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
@pytest.mark.parametrize("hash_routing", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 37])
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
num_experts, top_k, vocab = 16, 4, 64
scaling = 2.5
device = "cuda"
torch.manual_seed(0)
class Gate(torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.randn(num_experts, HIDDEN, device=device) * 0.1
self.e_score_correction_bias = (
None if hash_routing else torch.randn(num_experts, device=device)
)
self.tid2eid = (
torch.stack(
[
torch.randperm(num_experts, device=device)[:top_k]
for _ in range(vocab)
]
).to(torch.int32)
if hash_routing
else None
)
self.bias_vl = None
def forward(self, x):
return x.float() @ self.weight.t(), None
captured = {}
def experts(x, topk_weights, topk_ids):
captured["weights"], captured["ids"] = topk_weights, topk_ids
return 2 * x
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.gate = Gate()
moe.experts = experts
moe.shared_experts = lambda x: 3 * x
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = top_k
moe.renormalize = True
moe.routed_scaling_factor = scaling
moe.image_sentinel_lo = 0
x = torch.randn(num_tokens, HIDDEN, device=device, dtype=torch.bfloat16)
input_ids = torch.randint(0, vocab, (num_tokens,), device=device)
out = moe._forward_mega_moe(x, input_ids)
ids, weights = captured["ids"], captured["weights"]
assert ids.dtype == torch.int32 and ids.is_contiguous()
assert weights.dtype == torch.float32 and weights.is_contiguous()
scores = torch.nn.functional.softplus(x.float() @ moe.gate.weight.t()).sqrt()
if hash_routing:
expected_ids = moe.gate.tid2eid[input_ids].long()
else:
expected_ids = (scores + moe.gate.e_score_correction_bias).topk(top_k).indices
expected_weights = scores.gather(1, expected_ids)
expected_weights *= scaling / expected_weights.sum(dim=-1, keepdim=True)
order = ids.long().sort(dim=-1).indices
expected_order = expected_ids.sort(dim=-1).indices
torch.testing.assert_close(
ids.long().gather(1, order), expected_ids.gather(1, expected_order)
)
torch.testing.assert_close(
weights.gather(1, order),
expected_weights.gather(1, expected_order),
rtol=1e-3,
atol=1e-4,
)
torch.testing.assert_close(out, 5 * x)
+3 -1
View File
@@ -7,7 +7,7 @@ import pytest
import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.config import KernelConfig, ParallelConfig
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
@@ -58,6 +58,7 @@ def test_rocm_moe_routing_and_shared_experts_match_reference(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
parallel_config=ParallelConfig(),
kernel_config=default_vllm_config.kernel_config,
),
prefix="model.layers.0.ffn",
)
@@ -230,6 +231,7 @@ def test_rocm_moe_wires_vision_routing_on_hash_and_regular_layers(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
parallel_config=ParallelConfig(),
kernel_config=KernelConfig(),
)
hash_moe = rocm_model.DeepseekV4MoE(vllm_config, prefix="model.layers.0.ffn")