[ROCm][DSv4] AITER MegaMoEV2 Integration For DeepSeek V4 (#59685)
Signed-off-by: Micah Williamson <micah.williamson@amd.com> Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
co-authored by
Fangzhou Ai
parent
c83935b300
commit
f6e3944e31
@@ -5858,6 +5858,7 @@ steps:
|
||||
timeout_in_minutes: 20
|
||||
source_file_dependencies:
|
||||
- tests/models/test_deepseek_v4_vl_rocm.py
|
||||
- tests/models/test_deepseek_v4_aiter_mega_moe.py
|
||||
- vllm/models/deepseek_v4/amd/
|
||||
- vllm/models/deepseek_v4/common/
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
@@ -5871,6 +5872,7 @@ steps:
|
||||
- vllm/envs.py
|
||||
commands:
|
||||
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
|
||||
- pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py
|
||||
dind: false
|
||||
working_dir: /vllm-workspace/tests
|
||||
agent_pool: mi355_1
|
||||
|
||||
@@ -155,6 +155,7 @@ steps:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- tests/models/test_deepseek_v4_vl_rocm.py
|
||||
- tests/models/test_deepseek_v4_aiter_mega_moe.py
|
||||
- vllm/models/deepseek_v4/amd/
|
||||
- vllm/models/deepseek_v4/common/
|
||||
- vllm/model_executor/layers/fused_moe/
|
||||
@@ -170,6 +171,7 @@ steps:
|
||||
# Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror.
|
||||
# Pin the tested BF16 Triton routing and shared-expert path before imports.
|
||||
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
|
||||
- pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py
|
||||
|
||||
# Own job so mHC never shares a process with FlashInfer's import-time
|
||||
# ptxas override.
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd import mega_moe as mega_moe_module
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import (
|
||||
DeepseekV4AiterMegaMoEExperts,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN = 64
|
||||
INTER = 64
|
||||
|
||||
|
||||
def _make_experts(
|
||||
monkeypatch,
|
||||
*,
|
||||
ep_rank=0,
|
||||
ep_size=1,
|
||||
num_experts=2,
|
||||
intermediate_size=INTER,
|
||||
max_num_batched_tokens=3000,
|
||||
swiglu_limit=None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
mega_moe_module,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(
|
||||
rank_in_group=ep_rank, world_size=ep_size, cpu_group="ep"
|
||||
),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens)
|
||||
)
|
||||
return DeepseekV4AiterMegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=num_experts,
|
||||
top_k=2,
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size=intermediate_size,
|
||||
swiglu_limit=swiglu_limit,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_experts_load_into_local_slots(monkeypatch):
|
||||
model = torch.nn.Module()
|
||||
model.ffn = torch.nn.Module()
|
||||
model.ffn.experts = _make_experts(monkeypatch, ep_rank=1, ep_size=2, num_experts=4)
|
||||
params = dict(model.named_parameters())
|
||||
mapping = fused_moe_make_expert_params_mapping(
|
||||
model,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=4,
|
||||
routed_experts_prefix="",
|
||||
)
|
||||
|
||||
shapes = {
|
||||
("w1", "weight"): (INTER, HIDDEN // 2),
|
||||
("w3", "weight"): (INTER, HIDDEN // 2),
|
||||
("w2", "weight"): (HIDDEN, INTER // 2),
|
||||
("w1", "weight_scale"): (INTER, HIDDEN // 32),
|
||||
("w3", "weight_scale"): (INTER, HIDDEN // 32),
|
||||
("w2", "weight_scale"): (HIDDEN, INTER // 32),
|
||||
}
|
||||
shard_value = {"w1": 1, "w3": 2, "w2": 3}
|
||||
loaded = {}
|
||||
for expert in range(4):
|
||||
for (shard, suffix), shape in shapes.items():
|
||||
name = f"ffn.experts.{expert}.{shard}.{suffix}"
|
||||
weight = torch.full(
|
||||
shape, 10 * expert + shard_value[shard], dtype=torch.uint8
|
||||
)
|
||||
for param_name, weight_name, expert_id, shard_id in mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
mapped = name.replace(weight_name, param_name)
|
||||
param = params[mapped]
|
||||
if param.weight_loader(
|
||||
param,
|
||||
weight,
|
||||
mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
return_success=True,
|
||||
):
|
||||
loaded[name] = mapped
|
||||
break
|
||||
|
||||
assert sorted({n.split(".")[2] for n in loaded}) == ["2", "3"]
|
||||
assert len(loaded) == 2 * len(shapes)
|
||||
for slot, expert in enumerate((2, 3)):
|
||||
for suffix in ("weight", "weight_scale"):
|
||||
w13 = params[f"ffn.experts.w13_{suffix}"][slot]
|
||||
w2 = params[f"ffn.experts.w2_{suffix}"][slot]
|
||||
assert torch.all(w13[:INTER] == 10 * expert + 1)
|
||||
assert torch.all(w13[INTER:] == 10 * expert + 2)
|
||||
assert torch.all(w2 == 10 * expert + 3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
|
||||
@pytest.mark.parametrize("hash_routing", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 37])
|
||||
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
|
||||
num_experts, top_k, vocab = 16, 4, 64
|
||||
scaling = 2.5
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
class Gate(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = torch.randn(num_experts, HIDDEN, device=device) * 0.1
|
||||
self.e_score_correction_bias = (
|
||||
None if hash_routing else torch.randn(num_experts, device=device)
|
||||
)
|
||||
self.tid2eid = (
|
||||
torch.stack(
|
||||
[
|
||||
torch.randperm(num_experts, device=device)[:top_k]
|
||||
for _ in range(vocab)
|
||||
]
|
||||
).to(torch.int32)
|
||||
if hash_routing
|
||||
else None
|
||||
)
|
||||
self.bias_vl = None
|
||||
|
||||
def forward(self, x):
|
||||
return x.float() @ self.weight.t(), None
|
||||
|
||||
captured = {}
|
||||
|
||||
def experts(x, topk_weights, topk_ids):
|
||||
captured["weights"], captured["ids"] = topk_weights, topk_ids
|
||||
return 2 * x
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.gate = Gate()
|
||||
moe.experts = experts
|
||||
moe.shared_experts = lambda x: 3 * x
|
||||
moe.scoring_func = "sqrtsoftplus"
|
||||
moe.n_activated_experts = top_k
|
||||
moe.renormalize = True
|
||||
moe.routed_scaling_factor = scaling
|
||||
moe.image_sentinel_lo = 0
|
||||
|
||||
x = torch.randn(num_tokens, HIDDEN, device=device, dtype=torch.bfloat16)
|
||||
input_ids = torch.randint(0, vocab, (num_tokens,), device=device)
|
||||
out = moe._forward_mega_moe(x, input_ids)
|
||||
|
||||
ids, weights = captured["ids"], captured["weights"]
|
||||
assert ids.dtype == torch.int32 and ids.is_contiguous()
|
||||
assert weights.dtype == torch.float32 and weights.is_contiguous()
|
||||
|
||||
scores = torch.nn.functional.softplus(x.float() @ moe.gate.weight.t()).sqrt()
|
||||
if hash_routing:
|
||||
expected_ids = moe.gate.tid2eid[input_ids].long()
|
||||
else:
|
||||
expected_ids = (scores + moe.gate.e_score_correction_bias).topk(top_k).indices
|
||||
expected_weights = scores.gather(1, expected_ids)
|
||||
expected_weights *= scaling / expected_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
order = ids.long().sort(dim=-1).indices
|
||||
expected_order = expected_ids.sort(dim=-1).indices
|
||||
torch.testing.assert_close(
|
||||
ids.long().gather(1, order), expected_ids.gather(1, expected_order)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights.gather(1, order),
|
||||
expected_weights.gather(1, expected_order),
|
||||
rtol=1e-3,
|
||||
atol=1e-4,
|
||||
)
|
||||
torch.testing.assert_close(out, 5 * x)
|
||||
@@ -7,7 +7,7 @@ import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.config import KernelConfig, ParallelConfig
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -58,6 +58,7 @@ def test_rocm_moe_routing_and_shared_experts_match_reference(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
kernel_config=default_vllm_config.kernel_config,
|
||||
),
|
||||
prefix="model.layers.0.ffn",
|
||||
)
|
||||
@@ -230,6 +231,7 @@ def test_rocm_moe_wires_vision_routing_on_hash_and_regular_layers(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
parallel_config=ParallelConfig(),
|
||||
kernel_config=KernelConfig(),
|
||||
)
|
||||
|
||||
hash_moe = rocm_model.DeepseekV4MoE(vllm_config, prefix="model.layers.0.ffn")
|
||||
|
||||
@@ -137,6 +137,7 @@ MoEBackend = Literal[
|
||||
"humming",
|
||||
"triton_unfused",
|
||||
"aiter",
|
||||
"aiter_mega_moe",
|
||||
"aiter_triton_mxfp4_bf16",
|
||||
"flydsl",
|
||||
"hpc",
|
||||
@@ -317,6 +318,9 @@ class KernelConfig:
|
||||
- "humming": Use Humming Mixed Precision kernels
|
||||
- "triton_unfused": Use Triton unfused MoE kernels
|
||||
- "aiter": Use AMD AITer kernels (ROCm only)
|
||||
- "aiter_mega_moe": Use the AITER MegaMoE fused dispatch/GEMM/combine
|
||||
kernel (ROCm gfx950, requires DP+EP with a MoRI all2all backend;
|
||||
DeepSeek-V4 only)
|
||||
- "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16
|
||||
(moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250)
|
||||
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
|
||||
|
||||
@@ -1008,6 +1008,11 @@ class MoriAll2AllManager(All2AllManagerBase):
|
||||
self.handle_cache = Cache()
|
||||
|
||||
torch._C._distributed_c10d._register_process_group("mori", cpu_group)
|
||||
if get_current_vllm_config().kernel_config.moe_backend == "aiter_mega_moe":
|
||||
# MegaMoEV2 places its dispatch/combine workspaces on the MoRI
|
||||
# symmetric heap, which defaults to 2 GB. MegaMoEV2 requires > 4GB
|
||||
heap_size = os.environ.setdefault("MORI_SHMEM_HEAP_SIZE", "8G")
|
||||
logger.info_once("AITER MegaMoE: MORI_SHMEM_HEAP_SIZE=%s", heap_size)
|
||||
mori.shmem.shmem_torch_process_group_init("mori")
|
||||
|
||||
def _make_all2all_kwargs(
|
||||
|
||||
@@ -11,9 +11,8 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
|
||||
tilelang / triton / torch) instead of calling the tilelang kernels directly,
|
||||
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
|
||||
or TileLang fused MHC is available; False only on the torch fallback);
|
||||
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping``,
|
||||
``use_native_mega_moe`` and ``finalize_mega_moe_weights`` do not exist in
|
||||
amd/model.py).
|
||||
* use the AITER MegaMoE path (``aiter_mega_moe``) instead of the DeepGEMM
|
||||
mega-MoE path.
|
||||
|
||||
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
|
||||
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
|
||||
@@ -50,6 +49,7 @@ from vllm.model_executor.models.qwen3_dspark import (
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
from .mega_moe import finalize_mega_moe_weights
|
||||
from .model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
)
|
||||
@@ -385,14 +385,15 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
Non-mtp weights (embed/head/main layers) belong to the target model and
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
# AMD DeepseekV4MoE has no mega-MoE path; always use the standard
|
||||
# per-expert fused-MoE mapping (mirrors amd/mtp.py).
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
routed_experts_prefix=(
|
||||
"" if self.model.layers[0].ffn.use_mega_moe else "routed_experts"
|
||||
),
|
||||
)
|
||||
expert_scale_suffix = (
|
||||
".weight_scale"
|
||||
@@ -496,6 +497,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
finalize_mega_moe_weights(self)
|
||||
|
||||
def _remap_dspark_name(self, name: str) -> str | None:
|
||||
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
|
||||
class DeepseekV4AiterMegaMoEExperts(nn.Module):
|
||||
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
|
||||
# instance and passes its own weights per call.
|
||||
_runtime_cache: dict[tuple, Any] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
num_experts: int,
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
swiglu_limit: float | None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
ep_group = get_ep_group()
|
||||
self.ep_rank = ep_group.rank_in_group
|
||||
self.ep_size = ep_group.world_size
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_experts // self.ep_size
|
||||
self.experts_start_idx = self.ep_rank * self.num_local_experts
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.swiglu_limit = swiglu_limit or 0.0
|
||||
self.max_num_tokens = next_power_of_2(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def make_param(rows: int, cols: int) -> nn.Parameter:
|
||||
param = nn.Parameter(
|
||||
torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
set_weight_attrs(param, {"weight_loader": self.weight_loader})
|
||||
return param
|
||||
|
||||
self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2)
|
||||
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
|
||||
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
|
||||
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
|
||||
self._runtime: Any = None
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
local_expert_id = expert_id - self.experts_start_idx
|
||||
if not 0 <= local_expert_id < self.num_local_experts:
|
||||
return False if return_success else None
|
||||
expert_data = param.data[local_expert_id]
|
||||
if shard_id in ("w1", "w3"):
|
||||
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
|
||||
expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size)
|
||||
elif shard_id != "w2":
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
expert_data.copy_(loaded_weight.view(torch.uint8))
|
||||
return True if return_success else None
|
||||
|
||||
def finalize_weights(self) -> None:
|
||||
if self._runtime is not None:
|
||||
return
|
||||
n = self.num_local_experts
|
||||
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
self.w13_weight.data, 16, True
|
||||
)
|
||||
self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True
|
||||
)
|
||||
self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
|
||||
self.w2_weight.data, 16, False
|
||||
)
|
||||
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
|
||||
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
|
||||
)
|
||||
key = (
|
||||
id(get_ep_group().cpu_group),
|
||||
torch.accelerator.current_device_index(),
|
||||
self.num_experts,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.max_num_tokens,
|
||||
self.swiglu_limit,
|
||||
)
|
||||
runtime = self._runtime_cache.get(key)
|
||||
if runtime is None:
|
||||
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
|
||||
|
||||
runtime = MegaMoEV2(
|
||||
rank=self.ep_rank,
|
||||
world_size=self.ep_size,
|
||||
model_dim=self.hidden_size,
|
||||
inter_dim=self.intermediate_size,
|
||||
experts=self.num_experts,
|
||||
topk=self.top_k,
|
||||
quant="a8w4",
|
||||
w1=self.w13_weight,
|
||||
w1_scale=self.w13_weight_scale,
|
||||
w2=self.w2_weight,
|
||||
w2_scale=self.w2_weight_scale,
|
||||
max_tok_per_rank=self.max_num_tokens,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
)
|
||||
self._runtime_cache[key] = runtime
|
||||
self._runtime = runtime
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
runtime = self._runtime
|
||||
assert runtime is not None
|
||||
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
|
||||
runtime._s1_w1 = self.w13_weight
|
||||
runtime._s1_w1_scale = self.w13_weight_scale
|
||||
runtime.w2 = self.w2_weight
|
||||
runtime.w2_scale = self.w2_weight_scale
|
||||
return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids)
|
||||
|
||||
|
||||
DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
|
||||
|
||||
def finalize_mega_moe_weights(model: nn.Module) -> None:
|
||||
for module in model.modules():
|
||||
if isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
@@ -30,6 +30,9 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
@@ -71,6 +74,7 @@ from vllm.model_executor.models.utils import (
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts
|
||||
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
@@ -529,6 +533,39 @@ def _fuse_shared_experts_enabled(config, parallel_config: ParallelConfig) -> boo
|
||||
)
|
||||
|
||||
|
||||
def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
if not rocm_aiter_ops.is_fused_moe_enabled():
|
||||
raise ValueError(
|
||||
"--moe-backend aiter_mega_moe requires VLLM_ROCM_USE_AITER=1 "
|
||||
" and VLLM_ROCM_USE_AITER_MOE=1."
|
||||
)
|
||||
if not on_gfx950():
|
||||
raise NotImplementedError("AITER MegaMoE requires gfx950.")
|
||||
if (
|
||||
not parallel_config.enable_expert_parallel
|
||||
or parallel_config.tensor_parallel_size != 1
|
||||
or parallel_config.pipeline_parallel_size != 1
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"AITER MegaMoE requires data parallel attention with expert "
|
||||
"parallel MoE: --data-parallel-size N --enable-expert-parallel "
|
||||
"with TP=1 and PP=1."
|
||||
)
|
||||
if parallel_config.all2all_backend not in (
|
||||
"mori_high_throughput",
|
||||
"mori_low_latency",
|
||||
):
|
||||
raise ValueError(
|
||||
"AITER MegaMoE runs on MoRI shmem; add "
|
||||
"--all2all-backend mori_high_throughput."
|
||||
)
|
||||
if parallel_config.enable_eplb:
|
||||
raise NotImplementedError("AITER MegaMoE does not support EPLB.")
|
||||
if getattr(vllm_config.model_config.hf_config, "expert_dtype", "fp4") != "fp4":
|
||||
raise NotImplementedError("AITER MegaMoE only supports fp4 experts.")
|
||||
|
||||
|
||||
class DeepseekV4MoE(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -643,6 +680,20 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.experts_start_idx = self.tp_rank * self.n_local_experts
|
||||
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
|
||||
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe"
|
||||
if self.use_mega_moe:
|
||||
_validate_aiter_mega_moe_config(vllm_config)
|
||||
self.experts = DeepseekV4AiterMegaMoEExperts(
|
||||
vllm_config,
|
||||
num_experts=config.n_routed_experts,
|
||||
top_k=config.num_experts_per_tok,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=config.moe_intermediate_size,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
return
|
||||
|
||||
fuse_shared_into_routed = (
|
||||
self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert
|
||||
)
|
||||
@@ -692,6 +743,8 @@ class DeepseekV4MoE(nn.Module):
|
||||
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
if self.use_mega_moe:
|
||||
return self._forward_mega_moe(hidden_states, input_ids).view(org_shape)
|
||||
final_hidden_states = self.experts(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=hidden_states,
|
||||
@@ -700,6 +753,30 @@ class DeepseekV4MoE(nn.Module):
|
||||
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
def _forward_mega_moe(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
router_logits, _ = self.gate(hidden_states)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=router_logits,
|
||||
scoring_func=self.scoring_func,
|
||||
e_score_correction_bias=self.gate.e_score_correction_bias,
|
||||
topk=self.n_activated_experts,
|
||||
renormalize=self.renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=self.gate.tid2eid,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
bias_vl=self.gate.bias_vl,
|
||||
image_sentinel_lo=self.image_sentinel_lo,
|
||||
)
|
||||
final_hidden_states = self.experts(hidden_states, topk_weights, topk_ids)
|
||||
if self.shared_experts is not None:
|
||||
final_hidden_states = final_hidden_states + self.shared_experts(
|
||||
hidden_states
|
||||
)
|
||||
return final_hidden_states
|
||||
|
||||
|
||||
# Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm.
|
||||
_AITER_MHC_FUSED_RMSNORM_SIZES = frozenset({1280, 2560, 4096, 7168})
|
||||
@@ -993,6 +1070,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled(
|
||||
vllm_config
|
||||
)
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe"
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV4DecoderLayer(
|
||||
@@ -1315,6 +1393,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=num_experts,
|
||||
routed_experts_prefix="" if self.use_mega_moe else "routed_experts",
|
||||
)
|
||||
|
||||
|
||||
@@ -1453,6 +1532,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
|
||||
module.prepare_attn_preshuffle()
|
||||
elif isinstance(module, DeepseekV4MLP):
|
||||
module.prepare_gateup_preshuffle()
|
||||
elif isinstance(module, DeepseekV4AiterMegaMoEExperts):
|
||||
module.finalize_weights()
|
||||
if fused_compressor_layers:
|
||||
logger.info(
|
||||
"Fused the C4 compressor GEMMs in %d DeepSeek V4 layers",
|
||||
|
||||
@@ -46,6 +46,7 @@ from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .mega_moe import finalize_mega_moe_weights
|
||||
from .model import DeepseekV4DecoderLayer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -392,12 +393,16 @@ class DeepSeekV4MTP(nn.Module):
|
||||
head_rank_end = n_local_head * (tp_rank + 1)
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name="w1",
|
||||
ckpt_down_proj_name="w2",
|
||||
ckpt_up_proj_name="w3",
|
||||
num_experts=self.config.n_routed_experts,
|
||||
routed_experts_prefix=(
|
||||
"" if first_layer.mtp_block.ffn.use_mega_moe else "routed_experts"
|
||||
),
|
||||
)
|
||||
|
||||
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
|
||||
@@ -526,6 +531,9 @@ class DeepSeekV4MTP(nn.Module):
|
||||
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
|
||||
return loaded_params
|
||||
|
||||
def process_weights_after_loading(self) -> None:
|
||||
finalize_mega_moe_weights(self)
|
||||
|
||||
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
|
||||
"""Rewrite the weight name to match the format of the original model.
|
||||
Add .mtp_block for modules in transformer layer block for spec layer
|
||||
|
||||
Reference in New Issue
Block a user