diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 8317919ce4..9e634ea02b 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -5858,6 +5858,7 @@ steps: timeout_in_minutes: 20 source_file_dependencies: - tests/models/test_deepseek_v4_vl_rocm.py + - tests/models/test_deepseek_v4_aiter_mega_moe.py - vllm/models/deepseek_v4/amd/ - vllm/models/deepseek_v4/common/ - vllm/model_executor/layers/fused_moe/ @@ -5871,6 +5872,7 @@ steps: - vllm/envs.py commands: - VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py + - pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py dind: false working_dir: /vllm-workspace/tests agent_pool: mi355_1 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 54e7215967..cfc628d382 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -155,6 +155,7 @@ steps: - image-build-amd source_file_dependencies: - tests/models/test_deepseek_v4_vl_rocm.py + - tests/models/test_deepseek_v4_aiter_mega_moe.py - vllm/models/deepseek_v4/amd/ - vllm/models/deepseek_v4/common/ - vllm/model_executor/layers/fused_moe/ @@ -170,6 +171,7 @@ steps: # Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror. # Pin the tested BF16 Triton routing and shared-expert path before imports. - VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py + - pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py # Own job so mHC never shares a process with FlashInfer's import-time # ptxas override. diff --git a/tests/models/test_deepseek_v4_aiter_mega_moe.py b/tests/models/test_deepseek_v4_aiter_mega_moe.py new file mode 100644 index 0000000000..a8f692e07b --- /dev/null +++ b/tests/models/test_deepseek_v4_aiter_mega_moe.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) +from vllm.models.deepseek_v4.amd import mega_moe as mega_moe_module +from vllm.models.deepseek_v4.amd.mega_moe import ( + DeepseekV4AiterMegaMoEExperts, +) +from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE +from vllm.platforms import current_platform + +HIDDEN = 64 +INTER = 64 + + +def _make_experts( + monkeypatch, + *, + ep_rank=0, + ep_size=1, + num_experts=2, + intermediate_size=INTER, + max_num_batched_tokens=3000, + swiglu_limit=None, +): + monkeypatch.setattr( + mega_moe_module, + "get_ep_group", + lambda: SimpleNamespace( + rank_in_group=ep_rank, world_size=ep_size, cpu_group="ep" + ), + ) + vllm_config = SimpleNamespace( + scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens) + ) + return DeepseekV4AiterMegaMoEExperts( + vllm_config, + num_experts=num_experts, + top_k=2, + hidden_size=HIDDEN, + intermediate_size=intermediate_size, + swiglu_limit=swiglu_limit, + ) + + +@pytest.mark.cpu_test +def test_experts_load_into_local_slots(monkeypatch): + model = torch.nn.Module() + model.ffn = torch.nn.Module() + model.ffn.experts = _make_experts(monkeypatch, ep_rank=1, ep_size=2, num_experts=4) + params = dict(model.named_parameters()) + mapping = fused_moe_make_expert_params_mapping( + model, + ckpt_gate_proj_name="w1", + ckpt_down_proj_name="w2", + ckpt_up_proj_name="w3", + num_experts=4, + routed_experts_prefix="", + ) + + shapes = { + ("w1", "weight"): (INTER, HIDDEN // 2), + ("w3", "weight"): (INTER, HIDDEN // 2), + ("w2", "weight"): (HIDDEN, INTER // 2), + ("w1", "weight_scale"): (INTER, HIDDEN // 32), + ("w3", "weight_scale"): (INTER, HIDDEN // 32), + ("w2", "weight_scale"): (HIDDEN, INTER // 32), + } + shard_value = {"w1": 1, "w3": 2, "w2": 3} + loaded = {} + for expert in range(4): + for (shard, suffix), shape in shapes.items(): + name = f"ffn.experts.{expert}.{shard}.{suffix}" + weight = torch.full( + shape, 10 * expert + shard_value[shard], dtype=torch.uint8 + ) + for param_name, weight_name, expert_id, shard_id in mapping: + if weight_name not in name: + continue + mapped = name.replace(weight_name, param_name) + param = params[mapped] + if param.weight_loader( + param, + weight, + mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ): + loaded[name] = mapped + break + + assert sorted({n.split(".")[2] for n in loaded}) == ["2", "3"] + assert len(loaded) == 2 * len(shapes) + for slot, expert in enumerate((2, 3)): + for suffix in ("weight", "weight_scale"): + w13 = params[f"ffn.experts.w13_{suffix}"][slot] + w2 = params[f"ffn.experts.w2_{suffix}"][slot] + assert torch.all(w13[:INTER] == 10 * expert + 1) + assert torch.all(w13[INTER:] == 10 * expert + 2) + assert torch.all(w2 == 10 * expert + 3) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm") +@pytest.mark.parametrize("hash_routing", [False, True]) +@pytest.mark.parametrize("num_tokens", [1, 37]) +def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens): + num_experts, top_k, vocab = 16, 4, 64 + scaling = 2.5 + device = "cuda" + torch.manual_seed(0) + + class Gate(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.randn(num_experts, HIDDEN, device=device) * 0.1 + self.e_score_correction_bias = ( + None if hash_routing else torch.randn(num_experts, device=device) + ) + self.tid2eid = ( + torch.stack( + [ + torch.randperm(num_experts, device=device)[:top_k] + for _ in range(vocab) + ] + ).to(torch.int32) + if hash_routing + else None + ) + self.bias_vl = None + + def forward(self, x): + return x.float() @ self.weight.t(), None + + captured = {} + + def experts(x, topk_weights, topk_ids): + captured["weights"], captured["ids"] = topk_weights, topk_ids + return 2 * x + + moe = DeepseekV4MoE.__new__(DeepseekV4MoE) + torch.nn.Module.__init__(moe) + moe.gate = Gate() + moe.experts = experts + moe.shared_experts = lambda x: 3 * x + moe.scoring_func = "sqrtsoftplus" + moe.n_activated_experts = top_k + moe.renormalize = True + moe.routed_scaling_factor = scaling + moe.image_sentinel_lo = 0 + + x = torch.randn(num_tokens, HIDDEN, device=device, dtype=torch.bfloat16) + input_ids = torch.randint(0, vocab, (num_tokens,), device=device) + out = moe._forward_mega_moe(x, input_ids) + + ids, weights = captured["ids"], captured["weights"] + assert ids.dtype == torch.int32 and ids.is_contiguous() + assert weights.dtype == torch.float32 and weights.is_contiguous() + + scores = torch.nn.functional.softplus(x.float() @ moe.gate.weight.t()).sqrt() + if hash_routing: + expected_ids = moe.gate.tid2eid[input_ids].long() + else: + expected_ids = (scores + moe.gate.e_score_correction_bias).topk(top_k).indices + expected_weights = scores.gather(1, expected_ids) + expected_weights *= scaling / expected_weights.sum(dim=-1, keepdim=True) + + order = ids.long().sort(dim=-1).indices + expected_order = expected_ids.sort(dim=-1).indices + torch.testing.assert_close( + ids.long().gather(1, order), expected_ids.gather(1, expected_order) + ) + torch.testing.assert_close( + weights.gather(1, order), + expected_weights.gather(1, expected_order), + rtol=1e-3, + atol=1e-4, + ) + torch.testing.assert_close(out, 5 * x) diff --git a/tests/models/test_deepseek_v4_vl_rocm.py b/tests/models/test_deepseek_v4_vl_rocm.py index 036efbaa42..762f1cd6c0 100644 --- a/tests/models/test_deepseek_v4_vl_rocm.py +++ b/tests/models/test_deepseek_v4_vl_rocm.py @@ -7,7 +7,7 @@ import pytest import torch from torch import nn -from vllm.config import ParallelConfig +from vllm.config import KernelConfig, ParallelConfig from vllm.model_executor.models.utils import WeightsMapper from vllm.platforms import current_platform @@ -58,6 +58,7 @@ def test_rocm_moe_routing_and_shared_experts_match_reference( model_config=SimpleNamespace(hf_config=config), quant_config=None, parallel_config=ParallelConfig(), + kernel_config=default_vllm_config.kernel_config, ), prefix="model.layers.0.ffn", ) @@ -230,6 +231,7 @@ def test_rocm_moe_wires_vision_routing_on_hash_and_regular_layers( model_config=SimpleNamespace(hf_config=config), quant_config=None, parallel_config=ParallelConfig(), + kernel_config=KernelConfig(), ) hash_moe = rocm_model.DeepseekV4MoE(vllm_config, prefix="model.layers.0.ffn") diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index b8bfca082a..fa3ecf5df9 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -137,6 +137,7 @@ MoEBackend = Literal[ "humming", "triton_unfused", "aiter", + "aiter_mega_moe", "aiter_triton_mxfp4_bf16", "flydsl", "hpc", @@ -317,6 +318,9 @@ class KernelConfig: - "humming": Use Humming Mixed Precision kernels - "triton_unfused": Use Triton unfused MoE kernels - "aiter": Use AMD AITer kernels (ROCm only) + - "aiter_mega_moe": Use the AITER MegaMoE fused dispatch/GEMM/combine + kernel (ROCm gfx950, requires DP+EP with a MoRI all2all backend; + DeepSeek-V4 only) - "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16 (moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250) - "flydsl": Use AMD FlyDSL kernels (ROCm only) diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index d8790dbf42..ae31c0b1d3 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -1008,6 +1008,11 @@ class MoriAll2AllManager(All2AllManagerBase): self.handle_cache = Cache() torch._C._distributed_c10d._register_process_group("mori", cpu_group) + if get_current_vllm_config().kernel_config.moe_backend == "aiter_mega_moe": + # MegaMoEV2 places its dispatch/combine workspaces on the MoRI + # symmetric heap, which defaults to 2 GB. MegaMoEV2 requires > 4GB + heap_size = os.environ.setdefault("MORI_SHMEM_HEAP_SIZE", "8G") + logger.info_once("AITER MegaMoE: MORI_SHMEM_HEAP_SIZE=%s", heap_size) mori.shmem.shmem_torch_process_group_init("mori") def _make_all2all_kwargs( diff --git a/vllm/models/deepseek_v4/amd/dspark.py b/vllm/models/deepseek_v4/amd/dspark.py index e0c69c0328..e62431d7b6 100644 --- a/vllm/models/deepseek_v4/amd/dspark.py +++ b/vllm/models/deepseek_v4/amd/dspark.py @@ -11,9 +11,8 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for tilelang / triton / torch) instead of calling the tilelang kernels directly, and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER or TileLang fused MHC is available; False only on the torch fallback); - * drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping``, - ``use_native_mega_moe`` and ``finalize_mega_moe_weights`` do not exist in - amd/model.py). + * use the AITER MegaMoE path (``aiter_mega_moe``) instead of the DeepGEMM + mega-MoE path. Everything else — the semi-autoregressive drafting hooks, the Markov head, the sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is @@ -50,6 +49,7 @@ from vllm.model_executor.models.qwen3_dspark import ( ) from vllm.model_executor.models.utils import maybe_prefix +from .mega_moe import finalize_mega_moe_weights from .model import ( DeepseekV4DecoderLayer, ) @@ -385,14 +385,15 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): Non-mtp weights (embed/head/main layers) belong to the target model and are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target. """ - # AMD DeepseekV4MoE has no mega-MoE path; always use the standard - # per-expert fused-MoE mapping (mirrors amd/mtp.py). expert_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=self.config.n_routed_experts, + routed_experts_prefix=( + "" if self.model.layers[0].ffn.use_mega_moe else "routed_experts" + ), ) expert_scale_suffix = ( ".weight_scale" @@ -496,6 +497,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module): logger.info_once("DSpark draft model loaded: %d params", len(loaded_params)) return loaded_params + def process_weights_after_loading(self) -> None: + finalize_mega_moe_weights(self) + def _remap_dspark_name(self, name: str) -> str | None: """Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path. diff --git a/vllm/models/deepseek_v4/amd/mega_moe.py b/vllm/models/deepseek_v4/amd/mega_moe.py new file mode 100644 index 0000000000..2fdff93351 --- /dev/null +++ b/vllm/models/deepseek_v4/amd/mega_moe.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4.""" + +from typing import Any + +import torch +import torch.nn as nn + +from vllm._aiter_ops import rocm_aiter_ops +from vllm.config import VllmConfig +from vllm.distributed import get_ep_group +from vllm.model_executor.utils import set_weight_attrs +from vllm.utils.math_utils import next_power_of_2 + + +class DeepseekV4AiterMegaMoEExperts(nn.Module): + # MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one + # instance and passes its own weights per call. + _runtime_cache: dict[tuple, Any] = {} + + def __init__( + self, + vllm_config: VllmConfig, + *, + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + swiglu_limit: float | None, + prefix: str = "", + ): + super().__init__() + ep_group = get_ep_group() + self.ep_rank = ep_group.rank_in_group + self.ep_size = ep_group.world_size + self.num_experts = num_experts + self.num_local_experts = num_experts // self.ep_size + self.experts_start_idx = self.ep_rank * self.num_local_experts + self.top_k = top_k + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.swiglu_limit = swiglu_limit or 0.0 + self.max_num_tokens = next_power_of_2( + vllm_config.scheduler_config.max_num_batched_tokens + ) + self.prefix = prefix + + def make_param(rows: int, cols: int) -> nn.Parameter: + param = nn.Parameter( + torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8), + requires_grad=False, + ) + set_weight_attrs(param, {"weight_loader": self.weight_loader}) + return param + + self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2) + self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32) + self.w2_weight = make_param(hidden_size, intermediate_size // 2) + self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32) + self._runtime: Any = None + + def weight_loader( + self, + param: nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ) -> bool | None: + local_expert_id = expert_id - self.experts_start_idx + if not 0 <= local_expert_id < self.num_local_experts: + return False if return_success else None + expert_data = param.data[local_expert_id] + if shard_id in ("w1", "w3"): + shard_offset = 0 if shard_id == "w1" else self.intermediate_size + expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size) + elif shard_id != "w2": + raise ValueError(f"Unsupported expert shard id: {shard_id}") + expert_data.copy_(loaded_weight.view(torch.uint8)) + return True if return_success else None + + def finalize_weights(self) -> None: + if self._runtime is not None: + return + n = self.num_local_experts + self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4( + self.w13_weight.data, 16, True + ) + self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4( + self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True + ) + self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4( + self.w2_weight.data, 16, False + ) + self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4( + self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False + ) + key = ( + id(get_ep_group().cpu_group), + torch.accelerator.current_device_index(), + self.num_experts, + self.top_k, + self.hidden_size, + self.intermediate_size, + self.max_num_tokens, + self.swiglu_limit, + ) + runtime = self._runtime_cache.get(key) + if runtime is None: + from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2 + + runtime = MegaMoEV2( + rank=self.ep_rank, + world_size=self.ep_size, + model_dim=self.hidden_size, + inter_dim=self.intermediate_size, + experts=self.num_experts, + topk=self.top_k, + quant="a8w4", + w1=self.w13_weight, + w1_scale=self.w13_weight_scale, + w2=self.w2_weight, + w2_scale=self.w2_weight_scale, + max_tok_per_rank=self.max_num_tokens, + swiglu_limit=self.swiglu_limit, + ) + self._runtime_cache[key] = runtime + self._runtime = runtime + + def forward( + self, + hidden_states: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ) -> torch.Tensor: + runtime = self._runtime + assert runtime is not None + # MegaMoEV2 reads these per launch; rebind to this layer's experts. + runtime._s1_w1 = self.w13_weight + runtime._s1_w1_scale = self.w13_weight_scale + runtime.w2 = self.w2_weight + runtime.w2_scale = self.w2_weight_scale + return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids) + + +DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined] + + +def finalize_mega_moe_weights(model: nn.Module) -> None: + for module in model.modules(): + if isinstance(module, DeepseekV4AiterMegaMoEExperts): + module.finalize_weights() diff --git a/vllm/models/deepseek_v4/amd/model.py b/vllm/models/deepseek_v4/amd/model.py index f8306dd12c..8b26fc0060 100644 --- a/vllm/models/deepseek_v4/amd/model.py +++ b/vllm/models/deepseek_v4/amd/model.py @@ -30,6 +30,9 @@ from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( rocm_aiter_fused_experts, ) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + fused_topk_bias, +) from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) @@ -71,6 +74,7 @@ from vllm.model_executor.models.utils import ( make_layers, maybe_prefix, ) +from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention from vllm.platforms import current_platform from vllm.platforms.rocm import on_gfx950 @@ -529,6 +533,39 @@ def _fuse_shared_experts_enabled(config, parallel_config: ParallelConfig) -> boo ) +def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None: + parallel_config = vllm_config.parallel_config + if not rocm_aiter_ops.is_fused_moe_enabled(): + raise ValueError( + "--moe-backend aiter_mega_moe requires VLLM_ROCM_USE_AITER=1 " + " and VLLM_ROCM_USE_AITER_MOE=1." + ) + if not on_gfx950(): + raise NotImplementedError("AITER MegaMoE requires gfx950.") + if ( + not parallel_config.enable_expert_parallel + or parallel_config.tensor_parallel_size != 1 + or parallel_config.pipeline_parallel_size != 1 + ): + raise NotImplementedError( + "AITER MegaMoE requires data parallel attention with expert " + "parallel MoE: --data-parallel-size N --enable-expert-parallel " + "with TP=1 and PP=1." + ) + if parallel_config.all2all_backend not in ( + "mori_high_throughput", + "mori_low_latency", + ): + raise ValueError( + "AITER MegaMoE runs on MoRI shmem; add " + "--all2all-backend mori_high_throughput." + ) + if parallel_config.enable_eplb: + raise NotImplementedError("AITER MegaMoE does not support EPLB.") + if getattr(vllm_config.model_config.hf_config, "expert_dtype", "fp4") != "fp4": + raise NotImplementedError("AITER MegaMoE only supports fp4 experts.") + + class DeepseekV4MoE(nn.Module): def __init__( self, @@ -643,6 +680,20 @@ class DeepseekV4MoE(nn.Module): self.experts_start_idx = self.tp_rank * self.n_local_experts self.experts_end_idx = self.experts_start_idx + self.n_local_experts + self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe" + if self.use_mega_moe: + _validate_aiter_mega_moe_config(vllm_config) + self.experts = DeepseekV4AiterMegaMoEExperts( + vllm_config, + num_experts=config.n_routed_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + swiglu_limit=self.swiglu_limit, + prefix=f"{prefix}.experts", + ) + return + fuse_shared_into_routed = ( self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert ) @@ -692,6 +743,8 @@ class DeepseekV4MoE(nn.Module): raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.") org_shape = hidden_states.shape + if self.use_mega_moe: + return self._forward_mega_moe(hidden_states, input_ids).view(org_shape) final_hidden_states = self.experts( hidden_states=hidden_states, router_logits=hidden_states, @@ -700,6 +753,30 @@ class DeepseekV4MoE(nn.Module): return final_hidden_states.view(org_shape) + def _forward_mega_moe( + self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None + ) -> torch.Tensor: + router_logits, _ = self.gate(hidden_states) + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=router_logits, + scoring_func=self.scoring_func, + e_score_correction_bias=self.gate.e_score_correction_bias, + topk=self.n_activated_experts, + renormalize=self.renormalize, + input_tokens=input_ids, + hash_indices_table=self.gate.tid2eid, + routed_scaling_factor=self.routed_scaling_factor, + bias_vl=self.gate.bias_vl, + image_sentinel_lo=self.image_sentinel_lo, + ) + final_hidden_states = self.experts(hidden_states, topk_weights, topk_ids) + if self.shared_experts is not None: + final_hidden_states = final_hidden_states + self.shared_experts( + hidden_states + ) + return final_hidden_states + # Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm. _AITER_MHC_FUSED_RMSNORM_SIZES = frozenset({1280, 2560, 4096, 7168}) @@ -993,6 +1070,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled( vllm_config ) + self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe" self.start_layer, self.end_layer, self.layers = make_layers( config.num_hidden_layers, lambda prefix: DeepseekV4DecoderLayer( @@ -1315,6 +1393,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=num_experts, + routed_experts_prefix="" if self.use_mega_moe else "routed_experts", ) @@ -1453,6 +1532,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3): module.prepare_attn_preshuffle() elif isinstance(module, DeepseekV4MLP): module.prepare_gateup_preshuffle() + elif isinstance(module, DeepseekV4AiterMegaMoEExperts): + module.finalize_weights() if fused_compressor_layers: logger.info( "Fused the C4 compressor GEMMs in %d DeepSeek V4 layers", diff --git a/vllm/models/deepseek_v4/amd/mtp.py b/vllm/models/deepseek_v4/amd/mtp.py index 48ab036563..3746d6e387 100644 --- a/vllm/models/deepseek_v4/amd/mtp.py +++ b/vllm/models/deepseek_v4/amd/mtp.py @@ -46,6 +46,7 @@ from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import ( from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from .mega_moe import finalize_mega_moe_weights from .model import DeepseekV4DecoderLayer logger = init_logger(__name__) @@ -392,12 +393,16 @@ class DeepSeekV4MTP(nn.Module): head_rank_end = n_local_head * (tp_rank + 1) # Pre-compute expert mapping ONCE. + first_layer = next(iter(self.model.layers.values())) expert_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", ckpt_up_proj_name="w3", num_experts=self.config.n_routed_experts, + routed_experts_prefix=( + "" if first_layer.mtp_block.ffn.use_mega_moe else "routed_experts" + ), ) # FP8 experts register ``..._weight_scale_inv`` (block_quant) while @@ -526,6 +531,9 @@ class DeepSeekV4MTP(nn.Module): logger.info_once("MTP draft model loaded: %d params", len(loaded_params)) return loaded_params + def process_weights_after_loading(self) -> None: + finalize_mega_moe_weights(self) + def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: """Rewrite the weight name to match the format of the original model. Add .mtp_block for modules in transformer layer block for spec layer