[ROCm][DSv4] AITER MegaMoEV2 Integration For DeepSeek V4 (#59685)

Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com>
This commit is contained in:
Micah Williamson
2026-10-07 09:35:53 -05:00
committed by GitHub
co-authored by Fangzhou Ai
parent c83935b300
commit f6e3944e31
10 changed files with 454 additions and 6 deletions
+2
View File
@@ -5858,6 +5858,7 @@ steps:
timeout_in_minutes: 20
source_file_dependencies:
- tests/models/test_deepseek_v4_vl_rocm.py
- tests/models/test_deepseek_v4_aiter_mega_moe.py
- vllm/models/deepseek_v4/amd/
- vllm/models/deepseek_v4/common/
- vllm/model_executor/layers/fused_moe/
@@ -5871,6 +5872,7 @@ steps:
- vllm/envs.py
commands:
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
- pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py
dind: false
working_dir: /vllm-workspace/tests
agent_pool: mi355_1
+2
View File
@@ -155,6 +155,7 @@ steps:
- image-build-amd
source_file_dependencies:
- tests/models/test_deepseek_v4_vl_rocm.py
- tests/models/test_deepseek_v4_aiter_mega_moe.py
- vllm/models/deepseek_v4/amd/
- vllm/models/deepseek_v4/common/
- vllm/model_executor/layers/fused_moe/
@@ -170,6 +171,7 @@ steps:
# Common qnorm/RoPE/cache kernels already run in the H100 AMD mirror.
# Pin the tested BF16 Triton routing and shared-expert path before imports.
- VLLM_ROCM_USE_AITER_MOE=0 pytest -v -s models/test_deepseek_v4_vl_rocm.py
- pytest -v -s models/test_deepseek_v4_aiter_mega_moe.py
# Own job so mHC never shares a process with FlashInfer's import-time
# ptxas override.
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.models.deepseek_v4.amd import mega_moe as mega_moe_module
from vllm.models.deepseek_v4.amd.mega_moe import (
DeepseekV4AiterMegaMoEExperts,
)
from vllm.models.deepseek_v4.amd.model import DeepseekV4MoE
from vllm.platforms import current_platform
HIDDEN = 64
INTER = 64
def _make_experts(
monkeypatch,
*,
ep_rank=0,
ep_size=1,
num_experts=2,
intermediate_size=INTER,
max_num_batched_tokens=3000,
swiglu_limit=None,
):
monkeypatch.setattr(
mega_moe_module,
"get_ep_group",
lambda: SimpleNamespace(
rank_in_group=ep_rank, world_size=ep_size, cpu_group="ep"
),
)
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=max_num_batched_tokens)
)
return DeepseekV4AiterMegaMoEExperts(
vllm_config,
num_experts=num_experts,
top_k=2,
hidden_size=HIDDEN,
intermediate_size=intermediate_size,
swiglu_limit=swiglu_limit,
)
@pytest.mark.cpu_test
def test_experts_load_into_local_slots(monkeypatch):
model = torch.nn.Module()
model.ffn = torch.nn.Module()
model.ffn.experts = _make_experts(monkeypatch, ep_rank=1, ep_size=2, num_experts=4)
params = dict(model.named_parameters())
mapping = fused_moe_make_expert_params_mapping(
model,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=4,
routed_experts_prefix="",
)
shapes = {
("w1", "weight"): (INTER, HIDDEN // 2),
("w3", "weight"): (INTER, HIDDEN // 2),
("w2", "weight"): (HIDDEN, INTER // 2),
("w1", "weight_scale"): (INTER, HIDDEN // 32),
("w3", "weight_scale"): (INTER, HIDDEN // 32),
("w2", "weight_scale"): (HIDDEN, INTER // 32),
}
shard_value = {"w1": 1, "w3": 2, "w2": 3}
loaded = {}
for expert in range(4):
for (shard, suffix), shape in shapes.items():
name = f"ffn.experts.{expert}.{shard}.{suffix}"
weight = torch.full(
shape, 10 * expert + shard_value[shard], dtype=torch.uint8
)
for param_name, weight_name, expert_id, shard_id in mapping:
if weight_name not in name:
continue
mapped = name.replace(weight_name, param_name)
param = params[mapped]
if param.weight_loader(
param,
weight,
mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
):
loaded[name] = mapped
break
assert sorted({n.split(".")[2] for n in loaded}) == ["2", "3"]
assert len(loaded) == 2 * len(shapes)
for slot, expert in enumerate((2, 3)):
for suffix in ("weight", "weight_scale"):
w13 = params[f"ffn.experts.w13_{suffix}"][slot]
w2 = params[f"ffn.experts.w2_{suffix}"][slot]
assert torch.all(w13[:INTER] == 10 * expert + 1)
assert torch.all(w13[INTER:] == 10 * expert + 2)
assert torch.all(w2 == 10 * expert + 3)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="Requires ROCm")
@pytest.mark.parametrize("hash_routing", [False, True])
@pytest.mark.parametrize("num_tokens", [1, 37])
def test_mega_moe_routing_matches_reference(monkeypatch, hash_routing, num_tokens):
num_experts, top_k, vocab = 16, 4, 64
scaling = 2.5
device = "cuda"
torch.manual_seed(0)
class Gate(torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.randn(num_experts, HIDDEN, device=device) * 0.1
self.e_score_correction_bias = (
None if hash_routing else torch.randn(num_experts, device=device)
)
self.tid2eid = (
torch.stack(
[
torch.randperm(num_experts, device=device)[:top_k]
for _ in range(vocab)
]
).to(torch.int32)
if hash_routing
else None
)
self.bias_vl = None
def forward(self, x):
return x.float() @ self.weight.t(), None
captured = {}
def experts(x, topk_weights, topk_ids):
captured["weights"], captured["ids"] = topk_weights, topk_ids
return 2 * x
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.gate = Gate()
moe.experts = experts
moe.shared_experts = lambda x: 3 * x
moe.scoring_func = "sqrtsoftplus"
moe.n_activated_experts = top_k
moe.renormalize = True
moe.routed_scaling_factor = scaling
moe.image_sentinel_lo = 0
x = torch.randn(num_tokens, HIDDEN, device=device, dtype=torch.bfloat16)
input_ids = torch.randint(0, vocab, (num_tokens,), device=device)
out = moe._forward_mega_moe(x, input_ids)
ids, weights = captured["ids"], captured["weights"]
assert ids.dtype == torch.int32 and ids.is_contiguous()
assert weights.dtype == torch.float32 and weights.is_contiguous()
scores = torch.nn.functional.softplus(x.float() @ moe.gate.weight.t()).sqrt()
if hash_routing:
expected_ids = moe.gate.tid2eid[input_ids].long()
else:
expected_ids = (scores + moe.gate.e_score_correction_bias).topk(top_k).indices
expected_weights = scores.gather(1, expected_ids)
expected_weights *= scaling / expected_weights.sum(dim=-1, keepdim=True)
order = ids.long().sort(dim=-1).indices
expected_order = expected_ids.sort(dim=-1).indices
torch.testing.assert_close(
ids.long().gather(1, order), expected_ids.gather(1, expected_order)
)
torch.testing.assert_close(
weights.gather(1, order),
expected_weights.gather(1, expected_order),
rtol=1e-3,
atol=1e-4,
)
torch.testing.assert_close(out, 5 * x)
+3 -1
View File
@@ -7,7 +7,7 @@ import pytest
import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.config import KernelConfig, ParallelConfig
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
@@ -58,6 +58,7 @@ def test_rocm_moe_routing_and_shared_experts_match_reference(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
parallel_config=ParallelConfig(),
kernel_config=default_vllm_config.kernel_config,
),
prefix="model.layers.0.ffn",
)
@@ -230,6 +231,7 @@ def test_rocm_moe_wires_vision_routing_on_hash_and_regular_layers(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
parallel_config=ParallelConfig(),
kernel_config=KernelConfig(),
)
hash_moe = rocm_model.DeepseekV4MoE(vllm_config, prefix="model.layers.0.ffn")
+4
View File
@@ -137,6 +137,7 @@ MoEBackend = Literal[
"humming",
"triton_unfused",
"aiter",
"aiter_mega_moe",
"aiter_triton_mxfp4_bf16",
"flydsl",
"hpc",
@@ -317,6 +318,9 @@ class KernelConfig:
- "humming": Use Humming Mixed Precision kernels
- "triton_unfused": Use Triton unfused MoE kernels
- "aiter": Use AMD AITer kernels (ROCm only)
- "aiter_mega_moe": Use the AITER MegaMoE fused dispatch/GEMM/combine
kernel (ROCm gfx950, requires DP+EP with a MoRI all2all backend;
DeepSeek-V4 only)
- "aiter_triton_mxfp4_bf16": Use the AITER Triton MXFP4 W4A16
(moe_gemm_a16w4) MoE kernel (ROCm gfx942/gfx950/gfx1250)
- "flydsl": Use AMD FlyDSL kernels (ROCm only)
@@ -1008,6 +1008,11 @@ class MoriAll2AllManager(All2AllManagerBase):
self.handle_cache = Cache()
torch._C._distributed_c10d._register_process_group("mori", cpu_group)
if get_current_vllm_config().kernel_config.moe_backend == "aiter_mega_moe":
# MegaMoEV2 places its dispatch/combine workspaces on the MoRI
# symmetric heap, which defaults to 2 GB. MegaMoEV2 requires > 4GB
heap_size = os.environ.setdefault("MORI_SHMEM_HEAP_SIZE", "8G")
logger.info_once("AITER MegaMoE: MORI_SHMEM_HEAP_SIZE=%s", heap_size)
mori.shmem.shmem_torch_process_group_init("mori")
def _make_all2all_kwargs(
+9 -5
View File
@@ -11,9 +11,8 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
tilelang / triton / torch) instead of calling the tilelang kernels directly,
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
or TileLang fused MHC is available; False only on the torch fallback);
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping``,
``use_native_mega_moe`` and ``finalize_mega_moe_weights`` do not exist in
amd/model.py).
* use the AITER MegaMoE path (``aiter_mega_moe``) instead of the DeepGEMM
mega-MoE path.
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
@@ -50,6 +49,7 @@ from vllm.model_executor.models.qwen3_dspark import (
)
from vllm.model_executor.models.utils import maybe_prefix
from .mega_moe import finalize_mega_moe_weights
from .model import (
DeepseekV4DecoderLayer,
)
@@ -385,14 +385,15 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
Non-mtp weights (embed/head/main layers) belong to the target model and
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
# AMD DeepseekV4MoE has no mega-MoE path; always use the standard
# per-expert fused-MoE mapping (mirrors amd/mtp.py).
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
routed_experts_prefix=(
"" if self.model.layers[0].ffn.use_mega_moe else "routed_experts"
),
)
expert_scale_suffix = (
".weight_scale"
@@ -496,6 +497,9 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
logger.info_once("DSpark draft model loaded: %d params", len(loaded_params))
return loaded_params
def process_weights_after_loading(self) -> None:
finalize_mega_moe_weights(self)
def _remap_dspark_name(self, name: str) -> str | None:
"""Map a checkpoint ``mtp.{i}.*`` name to this model's parameter path.
+154
View File
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""AITER MegaMoE experts (fused dispatch, MXFP4 GEMMs, combine) for DeepSeek V4."""
from typing import Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.model_executor.utils import set_weight_attrs
from vllm.utils.math_utils import next_power_of_2
class DeepseekV4AiterMegaMoEExperts(nn.Module):
# MegaMoEV2 owns large symmetric-heap buffers, so every layer shares one
# instance and passes its own weights per call.
_runtime_cache: dict[tuple, Any] = {}
def __init__(
self,
vllm_config: VllmConfig,
*,
num_experts: int,
top_k: int,
hidden_size: int,
intermediate_size: int,
swiglu_limit: float | None,
prefix: str = "",
):
super().__init__()
ep_group = get_ep_group()
self.ep_rank = ep_group.rank_in_group
self.ep_size = ep_group.world_size
self.num_experts = num_experts
self.num_local_experts = num_experts // self.ep_size
self.experts_start_idx = self.ep_rank * self.num_local_experts
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.swiglu_limit = swiglu_limit or 0.0
self.max_num_tokens = next_power_of_2(
vllm_config.scheduler_config.max_num_batched_tokens
)
self.prefix = prefix
def make_param(rows: int, cols: int) -> nn.Parameter:
param = nn.Parameter(
torch.zeros(self.num_local_experts, rows, cols, dtype=torch.uint8),
requires_grad=False,
)
set_weight_attrs(param, {"weight_loader": self.weight_loader})
return param
self.w13_weight = make_param(2 * intermediate_size, hidden_size // 2)
self.w13_weight_scale = make_param(2 * intermediate_size, hidden_size // 32)
self.w2_weight = make_param(hidden_size, intermediate_size // 2)
self.w2_weight_scale = make_param(hidden_size, intermediate_size // 32)
self._runtime: Any = None
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
local_expert_id = expert_id - self.experts_start_idx
if not 0 <= local_expert_id < self.num_local_experts:
return False if return_success else None
expert_data = param.data[local_expert_id]
if shard_id in ("w1", "w3"):
shard_offset = 0 if shard_id == "w1" else self.intermediate_size
expert_data = expert_data.narrow(0, shard_offset, self.intermediate_size)
elif shard_id != "w2":
raise ValueError(f"Unsupported expert shard id: {shard_id}")
expert_data.copy_(loaded_weight.view(torch.uint8))
return True if return_success else None
def finalize_weights(self) -> None:
if self._runtime is not None:
return
n = self.num_local_experts
self.w13_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
self.w13_weight.data, 16, True
)
self.w13_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w13_weight_scale.data.view(-1, self.hidden_size // 32), n, True
)
self.w2_weight.data = rocm_aiter_ops.shuffle_weight_a16w4(
self.w2_weight.data, 16, False
)
self.w2_weight_scale.data = rocm_aiter_ops.shuffle_scale_a16w4(
self.w2_weight_scale.data.view(-1, self.intermediate_size // 32), n, False
)
key = (
id(get_ep_group().cpu_group),
torch.accelerator.current_device_index(),
self.num_experts,
self.top_k,
self.hidden_size,
self.intermediate_size,
self.max_num_tokens,
self.swiglu_limit,
)
runtime = self._runtime_cache.get(key)
if runtime is None:
from aiter.ops.flydsl.kernels.mega_moe import MegaMoEV2
runtime = MegaMoEV2(
rank=self.ep_rank,
world_size=self.ep_size,
model_dim=self.hidden_size,
inter_dim=self.intermediate_size,
experts=self.num_experts,
topk=self.top_k,
quant="a8w4",
w1=self.w13_weight,
w1_scale=self.w13_weight_scale,
w2=self.w2_weight,
w2_scale=self.w2_weight_scale,
max_tok_per_rank=self.max_num_tokens,
swiglu_limit=self.swiglu_limit,
)
self._runtime_cache[key] = runtime
self._runtime = runtime
def forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
) -> torch.Tensor:
runtime = self._runtime
assert runtime is not None
# MegaMoEV2 reads these per launch; rebind to this layer's experts.
runtime._s1_w1 = self.w13_weight
runtime._s1_w1_scale = self.w13_weight_scale
runtime.w2 = self.w2_weight
runtime.w2_scale = self.w2_weight_scale
return runtime.forward(hidden_states.contiguous(), topk_weights, topk_ids)
DeepseekV4AiterMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
def finalize_mega_moe_weights(model: nn.Module) -> None:
for module in model.modules():
if isinstance(module, DeepseekV4AiterMegaMoEExperts):
module.finalize_weights()
+81
View File
@@ -30,6 +30,9 @@ from vllm.model_executor.layers.fused_moe import (
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
@@ -71,6 +74,7 @@ from vllm.model_executor.models.utils import (
make_layers,
maybe_prefix,
)
from vllm.models.deepseek_v4.amd.mega_moe import DeepseekV4AiterMegaMoEExperts
from vllm.models.deepseek_v4.amd.rocm import DeepseekV4ROCMAiterMLAAttention
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx950
@@ -529,6 +533,39 @@ def _fuse_shared_experts_enabled(config, parallel_config: ParallelConfig) -> boo
)
def _validate_aiter_mega_moe_config(vllm_config: VllmConfig) -> None:
parallel_config = vllm_config.parallel_config
if not rocm_aiter_ops.is_fused_moe_enabled():
raise ValueError(
"--moe-backend aiter_mega_moe requires VLLM_ROCM_USE_AITER=1 "
" and VLLM_ROCM_USE_AITER_MOE=1."
)
if not on_gfx950():
raise NotImplementedError("AITER MegaMoE requires gfx950.")
if (
not parallel_config.enable_expert_parallel
or parallel_config.tensor_parallel_size != 1
or parallel_config.pipeline_parallel_size != 1
):
raise NotImplementedError(
"AITER MegaMoE requires data parallel attention with expert "
"parallel MoE: --data-parallel-size N --enable-expert-parallel "
"with TP=1 and PP=1."
)
if parallel_config.all2all_backend not in (
"mori_high_throughput",
"mori_low_latency",
):
raise ValueError(
"AITER MegaMoE runs on MoRI shmem; add "
"--all2all-backend mori_high_throughput."
)
if parallel_config.enable_eplb:
raise NotImplementedError("AITER MegaMoE does not support EPLB.")
if getattr(vllm_config.model_config.hf_config, "expert_dtype", "fp4") != "fp4":
raise NotImplementedError("AITER MegaMoE only supports fp4 experts.")
class DeepseekV4MoE(nn.Module):
def __init__(
self,
@@ -643,6 +680,20 @@ class DeepseekV4MoE(nn.Module):
self.experts_start_idx = self.tp_rank * self.n_local_experts
self.experts_end_idx = self.experts_start_idx + self.n_local_experts
self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe"
if self.use_mega_moe:
_validate_aiter_mega_moe_config(vllm_config)
self.experts = DeepseekV4AiterMegaMoEExperts(
vllm_config,
num_experts=config.n_routed_experts,
top_k=config.num_experts_per_tok,
hidden_size=config.hidden_size,
intermediate_size=config.moe_intermediate_size,
swiglu_limit=self.swiglu_limit,
prefix=f"{prefix}.experts",
)
return
fuse_shared_into_routed = (
self.is_fused_shared_expert_enabled or self.fuse_heterogeneous_shared_expert
)
@@ -692,6 +743,8 @@ class DeepseekV4MoE(nn.Module):
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
org_shape = hidden_states.shape
if self.use_mega_moe:
return self._forward_mega_moe(hidden_states, input_ids).view(org_shape)
final_hidden_states = self.experts(
hidden_states=hidden_states,
router_logits=hidden_states,
@@ -700,6 +753,30 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
def _forward_mega_moe(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
) -> torch.Tensor:
router_logits, _ = self.gate(hidden_states)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
scoring_func=self.scoring_func,
e_score_correction_bias=self.gate.e_score_correction_bias,
topk=self.n_activated_experts,
renormalize=self.renormalize,
input_tokens=input_ids,
hash_indices_table=self.gate.tid2eid,
routed_scaling_factor=self.routed_scaling_factor,
bias_vl=self.gate.bias_vl,
image_sentinel_lo=self.image_sentinel_lo,
)
final_hidden_states = self.experts(hidden_states, topk_weights, topk_ids)
if self.shared_experts is not None:
final_hidden_states = final_hidden_states + self.shared_experts(
hidden_states
)
return final_hidden_states
# Hidden sizes supported by AITER mhc_pre_big_fuse_rmsnorm.
_AITER_MHC_FUSED_RMSNORM_SIZES = frozenset({1280, 2560, 4096, 7168})
@@ -993,6 +1070,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.fuse_heterogeneous_shared_expert = _heterogeneous_shared_expert_enabled(
vllm_config
)
self.use_mega_moe = vllm_config.kernel_config.moe_backend == "aiter_mega_moe"
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV4DecoderLayer(
@@ -1315,6 +1393,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=num_experts,
routed_experts_prefix="" if self.use_mega_moe else "routed_experts",
)
@@ -1453,6 +1532,8 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
module.prepare_attn_preshuffle()
elif isinstance(module, DeepseekV4MLP):
module.prepare_gateup_preshuffle()
elif isinstance(module, DeepseekV4AiterMegaMoEExperts):
module.finalize_weights()
if fused_compressor_layers:
logger.info(
"Fused the C4 compressor GEMMs in %d DeepSeek V4 layers",
+8
View File
@@ -46,6 +46,7 @@ from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .mega_moe import finalize_mega_moe_weights
from .model import DeepseekV4DecoderLayer
logger = init_logger(__name__)
@@ -392,12 +393,16 @@ class DeepSeekV4MTP(nn.Module):
head_rank_end = n_local_head * (tp_rank + 1)
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
expert_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.n_routed_experts,
routed_experts_prefix=(
"" if first_layer.mtp_block.ffn.use_mega_moe else "routed_experts"
),
)
# FP8 experts register ``..._weight_scale_inv`` (block_quant) while
@@ -526,6 +531,9 @@ class DeepSeekV4MTP(nn.Module):
logger.info_once("MTP draft model loaded: %d params", len(loaded_params))
return loaded_params
def process_weights_after_loading(self) -> None:
finalize_mega_moe_weights(self)
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer