[ROCm][DSv4] AITER MegaMoEV2 Integration For DeepSeek V4 (#59685)
Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
co-authored by
Fangzhou Ai
parent
c83935b300
commit
f6e3944e31
@@ -0,0 +1,186 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd import mega_moe as mega_moe_module
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
DeepseekV4AiterMegaMoEExperts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN = 64
|
||||
INTER = 64
|
||||
|
||||
|
||||
def _make_experts(
|
||||
monkeypatch,
|
||||
*,
|
||||
ep_rank=0,
|
||||
ep_size=1,
|
||||
num_experts=2,
|
||||
intermediate_size=INTER,
|
||||
max_num_batched_tokens=3000,
|
||||
swiglu_limit=None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
mega_moe_module,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(
|
||||
rank_in_group=ep_rank, world_size=ep_size, cpu_group="ep"
|
||||
),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens)
|
||||
)
|
||||
return DeepseekV4AiterMegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=num_experts,
|
||||
top_k=2,
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size=intermediate_size,
|
||||
swiglu_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_experts_load_into_local_slots(monkeypatch):
|
||||
model = torch.nn.Module()
|
||||
model.ffn = torch.nn.Module()
|
||||
model.ffn.experts = _make_experts(monkeypatch, ep_rank=1, ep_size=2, num_experts=4)
|
||||
params = dict(model.named_parameters())
|
||||
mapping = fused_moe_make_expert_params_mapping(
|
||||
model,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=4,
|
||||
routed_experts_prefix="",
|
||||
)
|
||||
|
||||
shapes = {
|
||||
("w1", "weight"): (INTER, HIDDEN // 2),
|
||||
("w3", "weight"): (INTER, HIDDEN // 2),
|
||||
("w2", "weight"): (HIDDEN, INTER // 2),
|
||||
("w1", "weight_scale"): (INTER, HIDDEN // 32),
|
||||
("w3", "weight_scale"): (INTER, HIDDEN // 32),
|
||||
("w2", "weight_scale"): (HIDDEN, INTER // 32),
|
||||
}
|
||||
shard_value = {"w1": 1, "w3": 2, "w2": 3}
|
||||
loaded = {}
|
||||
for expert in range(4):
|
||||
for (shard, suffix), shape in shapes.items():
|
||||
name = f"ffn.experts.{expert}.{shard}.{suffix}"
|
||||
weight = torch.full(
|
||||
shape, 10 * expert + shard_value[shard], dtype=torch.uint8
|
||||
)
|
||||
for param_name, weight_name, expert_id, shard_id in mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
param = params[mapped]
|
||||
if param.weight_loader(
|
||||
param,
|
||||
weight,
|
||||
mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
):
|
||||
loaded[name] = mapped
|
||||
break
|
||||
|
||||
assert sorted({n.split(".")[2] for n in loaded}) == ["2", "3"]
|
||||
assert len(loaded) == 2 * len(shapes)
|
||||
for slot, expert in enumerate((2, 3)):
|
||||
for suffix in ("weight", "weight_scale"):
|
||||
w13 = params[f"ffn.experts.w13_{suffix}"][slot]
|
||||
w2 = params[f"ffn.experts.w2_{suffix}"][slot]
|
||||
assert torch.all(w13[:INTER] == 10 * expert + 1)
|
||||
assert torch.all(w13[INTER:] == 10 * expert + 2)
|
||||
assert torch.all(w2 == 10 * expert + 3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
@pytest.mark.parametrize("hash_routing", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 37])
|
||||
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
|
||||
num_experts, top_k, vocab = 16, 4, 64
|
||||
scaling = 2.5
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
class Gate(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.randn(num_experts, HIDDEN, device=device) * 0.1
|
||||
self.e_score_correction_bias = (
|
||||
None if hash_routing else torch.randn(num_experts, device=device)
|
||||
)
|
||||
self.tid2eid = (
|
||||
torch.stack(
|
||||
[
|
||||
torch.randperm(num_experts, device=device)[:top_k]
|
||||
for _ in range(vocab)
|
||||
]
|
||||
).to(torch.int32)
|
||||
if hash_routing
|
||||
else None
|
||||
)
|
||||
self.bias_vl = None
|
||||
|
||||
def forward(self, x):
|
||||
return x.float() @ self.weight.t(), None
|
||||
|
||||
captured = {}
|
||||
|
||||
def experts(x, topk_weights, topk_ids):
|
||||
captured["weights"], captured["ids"] = topk_weights, topk_ids
|
||||
return 2 * x
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.gate = Gate()
|
||||
moe.experts = experts
|
||||
moe.shared_experts = lambda x: 3 * x
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = top_k
|
||||
moe.renormalize = True
|
||||
moe.routed_scaling_factor = scaling
|
||||
moe.image_sentinel_lo = 0
|
||||
|
||||
x = torch.randn(num_tokens, HIDDEN, device=device, dtype=torch.bfloat16)
|
||||
input_ids = torch.randint(0, vocab, (num_tokens,), device=device)
|
||||
out = moe._forward_mega_moe(x, input_ids)
|
||||
|
||||
ids, weights = captured["ids"], captured["weights"]
|
||||
assert ids.dtype == torch.int32 and ids.is_contiguous()
|
||||
assert weights.dtype == torch.float32 and weights.is_contiguous()
|
||||
|
||||
scores = torch.nn.functional.softplus(x.float() @ moe.gate.weight.t()).sqrt()
|
||||
if hash_routing:
|
||||
expected_ids = moe.gate.tid2eid[input_ids].long()
|
||||
else:
|
||||
expected_ids = (scores + moe.gate.e_score_correction_bias).topk(top_k).indices
|
||||
expected_weights = scores.gather(1, expected_ids)
|
||||
expected_weights *= scaling / expected_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
order = ids.long().sort(dim=-1).indices
|
||||
expected_order = expected_ids.sort(dim=-1).indices
|
||||
torch.testing.assert_close(
|
||||
ids.long().gather(1, order), expected_ids.gather(1, expected_order)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights.gather(1, order),
|
||||
expected_weights.gather(1, expected_order),
|
||||
rtol=1e-3,
|
||||
atol=1e-4,
|
||||
)
|
||||
torch.testing.assert_close(out, 5 * x)
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config import KernelConfig, ParallelConfig
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -58,6 +58,7 @@ def test_rocm_moe_routing_and_shared_experts_match_reference(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
kernel_config=default_vllm_config.kernel_config,
|
||||
),
|
||||
prefix="model.layers.0.ffn",
|
||||
)
|
||||
@@ -230,6 +231,7 @@ def test_rocm_moe_wires_vision_routing_on_hash_and_regular_layers(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
kernel_config=KernelConfig(),
|
||||
)
|
||||
|
||||
hash_moe = rocm_model.DeepseekV4MoE(vllm_config, prefix="model.layers.0.ffn")
|
||||
|
||||
Reference in New Issue
Block a user