[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
+3 -2
View File
@@ -11,8 +11,9 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
tilelang / triton / torch) instead of calling the tilelang kernels directly,
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
or TileLang fused MHC is available; False only on the torch fallback);
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping`` /
``use_mega_moe`` / ``finalize_mega_moe_weights`` do not exist in amd/model.py).
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping``,
``use_native_mega_moe`` and ``finalize_mega_moe_weights`` do not exist in
amd/model.py).
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
+7 -5
View File
@@ -17,7 +17,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -99,7 +99,9 @@ class DSparkDeepseekV4Model(nn.Module):
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
@@ -240,7 +242,7 @@ class DSparkDeepseekV4Model(nn.Module):
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
if self.use_native_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
@@ -437,8 +439,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
first_layer = self.model.layers[0]
use_mega_moe = first_layer.ffn.use_mega_moe
if use_mega_moe:
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
if use_native_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
-397
View File
@@ -1,397 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer ``moe_ep`` expert module for DeepSeek V4."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
from vllm.utils.flashinfer_moe_ep import (
build_fi_mega_layer,
ensure_fi_moe_ep_runtime,
fi_moe_ep_backend_spec,
make_fi_moe_ep_bootstrap,
mega_moe_weight_pack_from_params,
)
if TYPE_CHECKING:
from flashinfer.moe_ep import MoEEpMegaLayer
from vllm.config import VllmConfig
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MLP
_MOE_SKIP_PADDING: bool | None = None
def ckpt_uses_nvfp4_experts(vllm_config: VllmConfig) -> bool:
"""True when the loaded checkpoint quantizes experts with modelopt NVFP4
(e2m1 + fp8-e4m3 per-16 block scales + per-tensor weight_scale_2), i.e.
the recipe the nvfp4_cutedsl prequantized-weights path consumes verbatim.
DeepSeek-V4 NVFP4-expert checkpoints declare ``moe_quant_algo: NVFP4`` in
their ``quantization_config``, which DeepseekV4FP8Config surfaces as
``moe_quant_algo``.
"""
return getattr(vllm_config.quant_config, "moe_quant_algo", "") == "NVFP4"
def resolve_mega_moe_is_padding(num_tokens: int) -> torch.Tensor | None:
from vllm.forward_context import get_forward_context, is_forward_context_available
global _MOE_SKIP_PADDING
if _MOE_SKIP_PADDING is None:
import vllm.envs as envs
_MOE_SKIP_PADDING = bool(envs.VLLM_MOE_SKIP_PADDING)
if not _MOE_SKIP_PADDING or not is_forward_context_available():
return None
is_padding = get_forward_context().is_padding
if is_padding is None:
return None
return is_padding[:num_tokens]
def apply_mega_moe_routing_preprocess(
topk_ids: torch.Tensor,
*,
is_padding: torch.Tensor | None = None,
) -> torch.Tensor:
"""Padding-only routing preprocess (EPLB hooks go here later)."""
if is_padding is not None:
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
return topk_ids
def nvfp4_prequant_pack_and_alphas(
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w13_weight_scale_2: torch.Tensor, # (E_local, 2) fp32: [:,0]=gate(w1), [:,1]=up(w3)
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
w2_weight_scale_2: torch.Tensor, # (E_local,) fp32
*,
intermediate_size: int,
):
"""NVFP4 checkpoint params -> (MoEWeightPack, fc1_alpha, fc2_alpha).
Activation quantization is fully dynamic, so the checkpoint's static
``input_scale`` drops out and each GEMM's epilogue alpha reduces to the
weight's per-tensor ``weight_scale_2``.
fc1_alpha is one scalar per expert, but gate (w1) and up (w3) carry their
own ``weight_scale_2``. When they differ, the ratio is folded into the up
half's e4m3 block scales — only if the fold round-trips exactly, since a
lossy rescale would silently change the model.
"""
from flashinfer.moe_ep import MoEWeightPack
inter = intermediate_size
gate_s2 = w13_weight_scale_2[:, 0].float()
up_s2 = w13_weight_scale_2[:, 1].float()
if (gate_s2 <= 0).any() or (up_s2 <= 0).any() or (w2_weight_scale_2 <= 0).any():
raise ValueError(
"nvfp4 prequant: non-positive weight_scale_2 loaded — checkpoint "
"scale tensors missing or loader routed them wrong."
)
w13_scale = w13_weight_scale
if not torch.equal(gate_s2, up_s2):
ratio = up_s2 / gate_s2 # (E_local,)
up_sf = w13_scale[:, inter:, :].float()
folded = up_sf * ratio[:, None, None]
folded_e4m3 = folded.to(torch.float8_e4m3fn)
if not torch.equal(folded_e4m3.float(), folded):
raise ValueError(
"NVFP4 MoE cannot merge the gate and up weight_scale_2 values "
"because their ratio is not exactly representable in "
"float8_e4m3fn. Use a checkpoint quantized with a shared "
"gate/up weight_scale_2."
)
w13_scale = w13_scale.clone()
w13_scale[:, inter:, :] = folded_e4m3
fc1_alpha = gate_s2.clone().contiguous()
fc2_alpha = w2_weight_scale_2.float().clone().contiguous()
pack = MoEWeightPack(
w13=w13_weight,
w2=w2_weight,
w13_scale=w13_scale,
w2_scale=w2_weight_scale,
)
return pack, fc1_alpha, fc2_alpha
class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
"""Same weight layout/loader as the native mega experts, FI compute path."""
def __init__(
self,
vllm_config: VllmConfig,
*,
activation_clamp: float | None = None,
**kwargs: Any,
) -> None:
super().__init__(vllm_config, **kwargs)
self._vllm_config = vllm_config
self._activation_clamp = activation_clamp
self._mega_layer: MoEEpMegaLayer | None = None
self._fast_ctx: tuple[Any, Any, Any, int, bool] | None = None
self._epilogue_alphas: tuple[torch.Tensor, torch.Tensor] | None = None
self._nvfp4_prequant = ckpt_uses_nvfp4_experts(vllm_config)
if self._nvfp4_prequant:
megakernel = fi_moe_ep_backend_spec(
vllm_config.kernel_config.moe_backend
).megakernel
if megakernel != "nvfp4_cutedsl":
raise ValueError(
"NVFP4-quantized expert checkpoint requires "
"moe_backend=flashinfer_moe_ep_mega_cutedsl, got a "
f"backend using megakernel {megakernel!r} "
"(deep_gemm consumes the MXFP4 checkpoint instead)."
)
self._realloc_nvfp4_params()
def _realloc_nvfp4_params(self) -> None:
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
fp8-e4m3 per-16 block scales plus the per-tensor second-level
scales (weight_scale_2, input_scale) modelopt exports."""
n_e = self.num_local_experts
inter = self.intermediate_size
hidden = self.hidden_size
attrs = {"weight_loader": self.weight_loader}
def _param(shape: tuple, dtype: torch.dtype) -> nn.Parameter:
p = nn.Parameter(torch.zeros(*shape, dtype=dtype), requires_grad=False)
set_weight_attrs(p, attrs)
return p
self.w13_weight_scale = _param(
(n_e, 2 * inter, hidden // 16), torch.float8_e4m3fn
)
self.w13_weight_scale.quant_method = "block"
self.w2_weight_scale = _param((n_e, hidden, inter // 16), torch.float8_e4m3fn)
self.w2_weight_scale.quant_method = "block"
# (E, 2): column 0 = gate (w1), column 1 = up (w3).
self.w13_weight_scale_2 = _param((n_e, 2), torch.float32)
self.w2_weight_scale_2 = _param((n_e,), torch.float32)
# Loaded for completeness; unused — activation quant is dynamic
# (see nvfp4_prequant_pack_and_alphas).
self.w13_input_scale = _param((n_e, 2), torch.float32)
self.w2_input_scale = _param((n_e,), torch.float32)
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
# NVFP4 checkpoint second-level scalars route here; everything
# else (packed weights, block scales) matches the base layout.
if "weight_scale_2" in weight_name or "input_scale" in weight_name:
local_expert_ids = self._map_global_expert_id(expert_id)
if not local_expert_ids:
return False if return_success else None
value = loaded_weight.reshape(()).to(torch.float32)
for local_expert_id in local_expert_ids:
if shard_id in ("w1", "w3"):
if "w13_" not in weight_name:
return False if return_success else None
param.data[local_expert_id, 0 if shard_id == "w1" else 1] = value
elif shard_id == "w2":
if "w2_" not in weight_name:
return False if return_success else None
param.data[local_expert_id] = value
else:
raise ValueError(f"Unsupported expert shard id: {shard_id}")
return True if return_success else None
return super().weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success,
)
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
# The FlashInfer megakernel has no shared-expert fusion; the caller's
# serial shared MLP path handles shared_experts.
if self._mega_layer is not None:
return
if self.w13_weight is None:
return
self._check_runtime_supported()
ensure_fi_moe_ep_runtime(self._vllm_config)
if self._nvfp4_prequant:
# NVFP4 checkpoint: hand the packed weights + both scale
# planes straight to the backend (no dequant->requant);
# per-expert globals become fc1/fc2 epilogue alphas staged
# at every forward via MoEEpTensors.
weights, fc1_alpha, fc2_alpha = nvfp4_prequant_pack_and_alphas(
self.w13_weight.data,
self.w13_weight_scale.data,
self.w13_weight_scale_2.data,
self.w2_weight.data,
self.w2_weight_scale.data,
self.w2_weight_scale_2.data,
intermediate_size=self.intermediate_size,
)
self._epilogue_alphas = (fc1_alpha, fc2_alpha)
else:
weights = mega_moe_weight_pack_from_params(
self.w13_weight,
self.w13_weight_scale,
self.w2_weight,
self.w2_weight_scale,
megakernel=fi_moe_ep_backend_spec(
self._vllm_config.kernel_config.moe_backend
).megakernel,
)
self._mega_layer = build_fi_mega_layer(
make_fi_moe_ep_bootstrap(),
vllm_config=self._vllm_config,
num_experts=self.num_experts,
max_tokens_per_rank=self.max_num_tokens,
hidden_size=self.hidden_size,
intermediate_size=self.intermediate_size,
top_k=self.top_k,
activation_clamp=self._activation_clamp,
weights=weights,
)
del weights
# Allocate (or attach to) the pooled workspace before first
# forward so warmup/capture never hits the lazy path.
self._mega_layer._ensure_workspace()
self.w13_weight = None
self.w13_weight_scale = None
self.w2_weight = None
self.w2_weight_scale = None
if self._nvfp4_prequant:
self.w13_weight_scale_2 = None
self.w2_weight_scale_2 = None
self.w13_input_scale = None
self.w2_input_scale = None
# EPLB is rejected for these backends in validate_fi_moe_ep_config, so
# nothing constructs an EplbState and none of these run. They are kept
# as explicit errors rather than deleted because inheriting the native
# implementations would be worse: set_eplb_state would record a map
# this forward path never consults, silently splitting routing from
# weights, and get_expert_weights reads transformed-weight attributes
# the flashinfer path releases after preprocess.
_EPLB_UNSUPPORTED = (
"EPLB is not supported with the flashinfer moe_ep backends; "
"validate_fi_moe_ep_config should have rejected this "
"configuration at startup."
)
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
raise NotImplementedError(self._EPLB_UNSUPPORTED)
def get_expert_weights(self) -> list[torch.Tensor]:
raise NotImplementedError(self._EPLB_UNSUPPORTED)
def update_expert_map(self) -> None:
raise NotImplementedError(self._EPLB_UNSUPPORTED)
def forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
*,
activation_clamp: float | None,
fast_math: bool = True,
) -> torch.Tensor:
# fast_math is a native deep_gemm knob; the FI kernels have no
# equivalent toggle, so it is accepted for signature parity only.
if hidden_states.shape[0] > self.max_num_tokens:
raise ValueError(
f"DeepSeek V4 MegaMoE got {hidden_states.shape[0]} tokens, "
f"but the symmetric buffer was sized for {self.max_num_tokens}."
)
from flashinfer.moe_ep import MoEEpTensors
num_tokens = hidden_states.shape[0]
is_padding = resolve_mega_moe_is_padding(num_tokens)
topk_ids = apply_mega_moe_routing_preprocess(
topk_ids,
is_padding=is_padding,
)
# Fast path: after the first successful full forward the layer is
# immutable, so skip MoEEpMegaLayer.forward()'s per-call validation
# and go straight to the kernel backend's stage_inputs + compute.
alphas = self._epilogue_alphas
fc1_alpha = alphas[0] if alphas is not None else None
fc2_alpha = alphas[1] if alphas is not None else None
fast = self._fast_ctx
if fast is not None:
kernel, workspace, transformed, hidden_size, zero_copy = fast
t = MoEEpTensors(
hidden_states=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
fc1_alpha=fc1_alpha,
fc2_alpha=fc2_alpha,
)
kernel.stage_inputs(t, workspace, quantize_input=True)
if zero_copy:
# Zero-copy (cutedsl backends): consume the workspace [:n]
# view directly — valid under stream ordering until the next
# MoE layer's launch on the shared workspace.
return kernel.compute(workspace, transformed, output=None)
# deep_gemm_mega's compute() requires a real output tensor.
out = torch.empty(
num_tokens,
hidden_size,
dtype=torch.bfloat16,
device=hidden_states.device,
)
return kernel.compute(workspace, transformed, output=out)
ensure_fi_moe_ep_runtime(self._vllm_config)
self.finalize_weights()
assert self._mega_layer is not None
y = self._mega_layer.forward(
MoEEpTensors(
hidden_states=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
fc1_alpha=fc1_alpha,
fc2_alpha=fc2_alpha,
)
)
layer = self._mega_layer
if hidden_states.dtype == torch.bfloat16:
self._fast_ctx = (
layer._kernel,
layer._ensure_workspace(),
layer._transformed,
layer._fleet_params.token_hidden_size,
# zero-copy output views are a cutedsl-backend contract
layer._kernel.kernel_name() != "deep_gemm_mega",
)
return y
DeepseekV4MegaMoEExpertsFI.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
+33 -43
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_ep_group,
get_pp_group,
@@ -85,10 +85,6 @@ from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.flashinfer_moe_ep import (
is_fi_moe_ep_backend,
validate_fi_moe_ep_config,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
@@ -807,10 +803,11 @@ class DeepseekV4MoE(nn.Module):
self.use_sequence_parallel = use_sequence_parallel
self.reduce_results = reduce_results
moe_backend = vllm_config.kernel_config.moe_backend
validate_fi_moe_ep_config(vllm_config)
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
self.use_fi_mega_moe = is_fi_moe_ep_backend(moe_backend)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
self.use_native_mega_moe = moe_backend in NATIVE_MEGA_MOE_BACKENDS
if (
self.use_native_mega_moe
and not vllm_config.parallel_config.enable_expert_parallel
):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
"Enable it with --enable-expert-parallel, or pick a different "
@@ -832,15 +829,15 @@ class DeepseekV4MoE(nn.Module):
self.swiglu_limit = config.swiglu_limit
self.renormalize = config.norm_topk_prob
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
if self.use_mega_moe and self.scoring_func != "sqrtsoftplus":
if self.use_native_mega_moe and self.scoring_func != "sqrtsoftplus":
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only."
)
if self.use_mega_moe and not self.renormalize:
if self.use_native_mega_moe and not self.renormalize:
raise NotImplementedError(
"DeepSeek V4 MegaMoE requires normalized top-k probabilities."
)
if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
if self.use_native_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
raise NotImplementedError(
"DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype="
f"{config.expert_dtype!r}. Drop --kernel-config moe_backend="
@@ -864,7 +861,9 @@ class DeepseekV4MoE(nn.Module):
image_sentinel_lo if getattr(config, "vision_n_layers", 0) > 0 else 0
)
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
self.hash_indices_dtype = (
torch.int64 if self.use_native_mega_moe else torch.int32
)
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
# Use randint instead of empty to avoid garbage values causing
@@ -908,12 +907,12 @@ class DeepseekV4MoE(nn.Module):
hidden_act=config.hidden_act,
swiglu_limit=self.swiglu_limit,
quant_config=quant_config,
reduce_results=self.use_mega_moe,
reduce_results=self.use_native_mega_moe,
is_sequence_parallel=use_sequence_parallel,
prefix=f"{prefix}.shared_experts",
)
if self.use_mega_moe:
if self.use_native_mega_moe:
self._init_mega_moe_experts(vllm_config, config, prefix)
else:
self._init_fused_moe_experts(vllm_config, config, quant_config, prefix)
@@ -950,28 +949,13 @@ class DeepseekV4MoE(nn.Module):
# Native DeepGEMM fusion requires each EP rank to own the complete
# shared MLP. Sequence parallel replicates those weights while sharding
# tokens. TP=1 is also naturally replicated. With PP+TP the shared MLP
# remains tensor-sharded, so retain the serial path. The FlashInfer
# megakernel has no shared-expert fusion, so it keeps the serial path.
# remains tensor-sharded, so retain the serial path.
fuse_shared_experts = bool(
self.shared_experts is not None
and not envs.VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION
and (self.use_sequence_parallel or self.tp_size == 1)
and not self.use_fi_mega_moe
)
activation_clamp = (
float(self.swiglu_limit) if self.swiglu_limit is not None else None
)
if self.use_fi_mega_moe:
# Deferred: fi_moe subclasses DeepseekV4MegaMoEExperts, so a
# module-level import here would be circular.
from vllm.models.deepseek_v4.nvidia.fi_moe import (
DeepseekV4MegaMoEExpertsFI,
)
experts_cls: type[DeepseekV4MegaMoEExperts] = DeepseekV4MegaMoEExpertsFI
else:
experts_cls = DeepseekV4MegaMoEExperts
expert_kwargs: dict[str, typing.Any] = dict(
num_experts=self.n_physical_experts,
num_local_experts=self.n_local_physical_experts,
@@ -983,9 +967,7 @@ class DeepseekV4MoE(nn.Module):
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
prefix=f"{prefix}.experts",
)
if self.use_fi_mega_moe:
expert_kwargs["activation_clamp"] = activation_clamp
self.experts = experts_cls(vllm_config, **expert_kwargs)
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
def _init_fused_moe_experts(
self,
@@ -1048,7 +1030,7 @@ class DeepseekV4MoE(nn.Module):
if bias_vl is not None and input_ids is None:
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
if not self.use_mega_moe:
if not self.use_native_mega_moe:
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
@@ -1139,7 +1121,7 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
def finalize_mega_moe_weights(self) -> None:
if self.use_mega_moe:
if self.use_native_mega_moe:
self.experts.finalize_weights(self.shared_experts)
@@ -1179,12 +1161,15 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
parallel_config = vllm_config.parallel_config
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
moe_backend = vllm_config.kernel_config.moe_backend
moe_needs_token_sharded_input = (
moe_backend in MEGA_MOE_BACKENDS or parallel_config.data_parallel_size > 1
)
return (
parallel_config.pipeline_parallel_size == 1
and parallel_config.enable_expert_parallel
and parallel_config.tensor_parallel_size > 1
and (use_mega_moe or parallel_config.data_parallel_size > 1)
and moe_needs_token_sharded_input
)
@@ -1425,9 +1410,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.config = config
self.quant_config = quant_config
self.parallel_config = vllm_config.parallel_config
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
if (
self.use_native_mega_moe
and not vllm_config.parallel_config.enable_expert_parallel
):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
"Enable it with --enable-expert-parallel, or pick a different "
@@ -1564,7 +1554,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
if self.use_mega_moe:
if self.use_native_mega_moe:
input_ids = input_ids.to(torch.int64)
full_num_tokens = positions.shape[0]
@@ -1578,7 +1568,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
if self.use_native_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=self.has_local_hash_moe,
@@ -1802,7 +1792,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
if first_layer.ffn.use_mega_moe:
if first_layer.ffn.use_native_mega_moe:
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
+6 -4
View File
@@ -20,7 +20,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -95,7 +95,9 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
self.config = config
quant_config = vllm_config.quant_config
self.rms_norm_eps = config.rms_norm_eps
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -186,7 +188,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
inputs_embeds
).unsqueeze(-2)
mega_gate_metadata = None
if self.use_mega_moe:
if self.use_native_mega_moe:
routing_input_ids = input_ids
if self.mtp_block.use_sequence_parallel:
routing_input_ids = sp_shard(routing_input_ids)
@@ -384,7 +386,7 @@ class DeepSeekV4MTP(nn.Module):
# Pre-compute expert mapping ONCE.
first_layer = next(iter(self.model.layers.values()))
if first_layer.mtp_block.ffn.use_mega_moe:
if first_layer.mtp_block.ffn.use_native_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(
self.config.n_routed_experts
)
+7 -5
View File
@@ -18,7 +18,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -81,7 +81,9 @@ class DSparkDeepseekV4Model(nn.Module):
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = (
@@ -214,7 +216,7 @@ class DSparkDeepseekV4Model(nn.Module):
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
if self.use_native_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
@@ -371,7 +373,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
"""
first_layer = self.model.layers[0]
use_mega_moe = first_layer.ffn.use_mega_moe
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
# Draft MoE layers use the dspark_* expert counts, not the
# backbone's (see DeepseekV4MoE and the reference
# ModelArgs.get_moe_config).
@@ -379,7 +381,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
getattr(self.config, "dspark_n_routed_experts", 0)
or self.config.n_routed_experts
)
if use_mega_moe:
if use_native_mega_moe:
expert_mapping = make_deepseek_v4_expert_params_mapping(n_draft_experts)
else:
expert_mapping = fused_moe_make_expert_params_mapping(
+16 -9
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_engram_dp_size,
get_pp_group,
@@ -174,7 +174,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
def defers_finalize(self, num_tokens: int) -> bool:
return (
not self.use_mega_moe
not self.use_native_mega_moe
and self.experts.moe_config.should_defer_moe_finalize(num_tokens)
)
@@ -244,12 +244,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
parallel_config = vllm_config.parallel_config
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
moe_needs_token_sharded_input = (
vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
)
return (
parallel_config.pipeline_parallel_size == 1
and parallel_config.enable_expert_parallel
and parallel_config.tensor_parallel_size > 1
and (use_mega_moe or parallel_config.data_parallel_size > 1)
and (moe_needs_token_sharded_input or parallel_config.data_parallel_size > 1)
)
@@ -631,9 +633,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.config = config
self.quant_config = quant_config
self.parallel_config = vllm_config.parallel_config
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
if (
self.use_native_mega_moe
and not vllm_config.parallel_config.enable_expert_parallel
):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
"Enable it with --enable-expert-parallel, or pick a different "
@@ -865,7 +872,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
if self.use_mega_moe:
if self.use_native_mega_moe:
input_ids = input_ids.to(torch.int64)
# Engram n-gram hashes for the whole (flattened) batch, computed once
@@ -1015,7 +1022,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
def _mega_gate_metadata(
self, input_ids: torch.Tensor | None
) -> MegaGateRoutingMetadata | None:
if not self.use_mega_moe:
if not self.use_native_mega_moe:
return None
assert input_ids is not None
return prepare_mega_gate_routing_metadata(
@@ -1331,7 +1338,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
if first_layer.ffn.use_mega_moe:
if first_layer.ffn.use_native_mega_moe:
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)