[feat] FlashInfer CuteDSL MegaMoE integration (#54049)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -11,8 +11,9 @@ ROCm port of ``nvidia/dspark.py``. Follows the same nvidia->amd recipe used for
|
||||
tilelang / triton / torch) instead of calling the tilelang kernels directly,
|
||||
and gate the trailing ``mhc_post`` on ``use_fused_mhc`` (True when AITER
|
||||
or TileLang fused MHC is available; False only on the torch fallback);
|
||||
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping`` /
|
||||
``use_mega_moe`` / ``finalize_mega_moe_weights`` do not exist in amd/model.py).
|
||||
* drop the mega-MoE weight path (``make_deepseek_v4_expert_params_mapping``,
|
||||
``use_native_mega_moe`` and ``finalize_mega_moe_weights`` do not exist in
|
||||
amd/model.py).
|
||||
|
||||
Everything else — the semi-autoregressive drafting hooks, the Markov head, the
|
||||
sliding-window context-KV insert, and the checkpoint ``mtp.*`` weight remap — is
|
||||
|
||||
@@ -17,7 +17,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -99,7 +99,9 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
|
||||
@@ -240,7 +242,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
input_ids = sp_shard(input_ids)
|
||||
mega_gate_metadata = None
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
input_ids,
|
||||
has_hash_routing=False,
|
||||
@@ -437,8 +439,8 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
first_layer = self.model.layers[0]
|
||||
use_mega_moe = first_layer.ffn.use_mega_moe
|
||||
if use_mega_moe:
|
||||
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
|
||||
if use_native_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer ``moe_ep`` expert module for DeepSeek V4."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MegaMoEExperts
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
build_fi_mega_layer,
|
||||
ensure_fi_moe_ep_runtime,
|
||||
fi_moe_ep_backend_spec,
|
||||
make_fi_moe_ep_bootstrap,
|
||||
mega_moe_weight_pack_from_params,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from flashinfer.moe_ep import MoEEpMegaLayer
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.models.deepseek_v4.nvidia.model import DeepseekV4MLP
|
||||
|
||||
_MOE_SKIP_PADDING: bool | None = None
|
||||
|
||||
|
||||
def ckpt_uses_nvfp4_experts(vllm_config: VllmConfig) -> bool:
|
||||
"""True when the loaded checkpoint quantizes experts with modelopt NVFP4
|
||||
(e2m1 + fp8-e4m3 per-16 block scales + per-tensor weight_scale_2), i.e.
|
||||
the recipe the nvfp4_cutedsl prequantized-weights path consumes verbatim.
|
||||
|
||||
DeepSeek-V4 NVFP4-expert checkpoints declare ``moe_quant_algo: NVFP4`` in
|
||||
their ``quantization_config``, which DeepseekV4FP8Config surfaces as
|
||||
``moe_quant_algo``.
|
||||
"""
|
||||
return getattr(vllm_config.quant_config, "moe_quant_algo", "") == "NVFP4"
|
||||
|
||||
|
||||
def resolve_mega_moe_is_padding(num_tokens: int) -> torch.Tensor | None:
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
|
||||
global _MOE_SKIP_PADDING
|
||||
if _MOE_SKIP_PADDING is None:
|
||||
import vllm.envs as envs
|
||||
|
||||
_MOE_SKIP_PADDING = bool(envs.VLLM_MOE_SKIP_PADDING)
|
||||
if not _MOE_SKIP_PADDING or not is_forward_context_available():
|
||||
return None
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is None:
|
||||
return None
|
||||
return is_padding[:num_tokens]
|
||||
|
||||
|
||||
def apply_mega_moe_routing_preprocess(
|
||||
topk_ids: torch.Tensor,
|
||||
*,
|
||||
is_padding: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Padding-only routing preprocess (EPLB hooks go here later)."""
|
||||
if is_padding is not None:
|
||||
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
|
||||
return topk_ids
|
||||
|
||||
|
||||
def nvfp4_prequant_pack_and_alphas(
|
||||
w13_weight: torch.Tensor,
|
||||
w13_weight_scale: torch.Tensor,
|
||||
w13_weight_scale_2: torch.Tensor, # (E_local, 2) fp32: [:,0]=gate(w1), [:,1]=up(w3)
|
||||
w2_weight: torch.Tensor,
|
||||
w2_weight_scale: torch.Tensor,
|
||||
w2_weight_scale_2: torch.Tensor, # (E_local,) fp32
|
||||
*,
|
||||
intermediate_size: int,
|
||||
):
|
||||
"""NVFP4 checkpoint params -> (MoEWeightPack, fc1_alpha, fc2_alpha).
|
||||
|
||||
Activation quantization is fully dynamic, so the checkpoint's static
|
||||
``input_scale`` drops out and each GEMM's epilogue alpha reduces to the
|
||||
weight's per-tensor ``weight_scale_2``.
|
||||
|
||||
fc1_alpha is one scalar per expert, but gate (w1) and up (w3) carry their
|
||||
own ``weight_scale_2``. When they differ, the ratio is folded into the up
|
||||
half's e4m3 block scales — only if the fold round-trips exactly, since a
|
||||
lossy rescale would silently change the model.
|
||||
"""
|
||||
from flashinfer.moe_ep import MoEWeightPack
|
||||
|
||||
inter = intermediate_size
|
||||
gate_s2 = w13_weight_scale_2[:, 0].float()
|
||||
up_s2 = w13_weight_scale_2[:, 1].float()
|
||||
if (gate_s2 <= 0).any() or (up_s2 <= 0).any() or (w2_weight_scale_2 <= 0).any():
|
||||
raise ValueError(
|
||||
"nvfp4 prequant: non-positive weight_scale_2 loaded — checkpoint "
|
||||
"scale tensors missing or loader routed them wrong."
|
||||
)
|
||||
|
||||
w13_scale = w13_weight_scale
|
||||
if not torch.equal(gate_s2, up_s2):
|
||||
ratio = up_s2 / gate_s2 # (E_local,)
|
||||
up_sf = w13_scale[:, inter:, :].float()
|
||||
folded = up_sf * ratio[:, None, None]
|
||||
folded_e4m3 = folded.to(torch.float8_e4m3fn)
|
||||
if not torch.equal(folded_e4m3.float(), folded):
|
||||
raise ValueError(
|
||||
"NVFP4 MoE cannot merge the gate and up weight_scale_2 values "
|
||||
"because their ratio is not exactly representable in "
|
||||
"float8_e4m3fn. Use a checkpoint quantized with a shared "
|
||||
"gate/up weight_scale_2."
|
||||
)
|
||||
w13_scale = w13_scale.clone()
|
||||
w13_scale[:, inter:, :] = folded_e4m3
|
||||
|
||||
fc1_alpha = gate_s2.clone().contiguous()
|
||||
fc2_alpha = w2_weight_scale_2.float().clone().contiguous()
|
||||
pack = MoEWeightPack(
|
||||
w13=w13_weight,
|
||||
w2=w2_weight,
|
||||
w13_scale=w13_scale,
|
||||
w2_scale=w2_weight_scale,
|
||||
)
|
||||
return pack, fc1_alpha, fc2_alpha
|
||||
|
||||
|
||||
class DeepseekV4MegaMoEExpertsFI(DeepseekV4MegaMoEExperts):
|
||||
"""Same weight layout/loader as the native mega experts, FI compute path."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
activation_clamp: float | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, **kwargs)
|
||||
self._vllm_config = vllm_config
|
||||
self._activation_clamp = activation_clamp
|
||||
self._mega_layer: MoEEpMegaLayer | None = None
|
||||
self._fast_ctx: tuple[Any, Any, Any, int, bool] | None = None
|
||||
self._epilogue_alphas: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._nvfp4_prequant = ckpt_uses_nvfp4_experts(vllm_config)
|
||||
if self._nvfp4_prequant:
|
||||
megakernel = fi_moe_ep_backend_spec(
|
||||
vllm_config.kernel_config.moe_backend
|
||||
).megakernel
|
||||
if megakernel != "nvfp4_cutedsl":
|
||||
raise ValueError(
|
||||
"NVFP4-quantized expert checkpoint requires "
|
||||
"moe_backend=flashinfer_moe_ep_mega_cutedsl, got a "
|
||||
f"backend using megakernel {megakernel!r} "
|
||||
"(deep_gemm consumes the MXFP4 checkpoint instead)."
|
||||
)
|
||||
self._realloc_nvfp4_params()
|
||||
|
||||
def _realloc_nvfp4_params(self) -> None:
|
||||
"""Swap the mx-recipe scale params for the NVFP4 checkpoint's:
|
||||
fp8-e4m3 per-16 block scales plus the per-tensor second-level
|
||||
scales (weight_scale_2, input_scale) modelopt exports."""
|
||||
n_e = self.num_local_experts
|
||||
inter = self.intermediate_size
|
||||
hidden = self.hidden_size
|
||||
attrs = {"weight_loader": self.weight_loader}
|
||||
|
||||
def _param(shape: tuple, dtype: torch.dtype) -> nn.Parameter:
|
||||
p = nn.Parameter(torch.zeros(*shape, dtype=dtype), requires_grad=False)
|
||||
set_weight_attrs(p, attrs)
|
||||
return p
|
||||
|
||||
self.w13_weight_scale = _param(
|
||||
(n_e, 2 * inter, hidden // 16), torch.float8_e4m3fn
|
||||
)
|
||||
self.w13_weight_scale.quant_method = "block"
|
||||
self.w2_weight_scale = _param((n_e, hidden, inter // 16), torch.float8_e4m3fn)
|
||||
self.w2_weight_scale.quant_method = "block"
|
||||
# (E, 2): column 0 = gate (w1), column 1 = up (w3).
|
||||
self.w13_weight_scale_2 = _param((n_e, 2), torch.float32)
|
||||
self.w2_weight_scale_2 = _param((n_e,), torch.float32)
|
||||
# Loaded for completeness; unused — activation quant is dynamic
|
||||
# (see nvfp4_prequant_pack_and_alphas).
|
||||
self.w13_input_scale = _param((n_e, 2), torch.float32)
|
||||
self.w2_input_scale = _param((n_e,), torch.float32)
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
# NVFP4 checkpoint second-level scalars route here; everything
|
||||
# else (packed weights, block scales) matches the base layout.
|
||||
if "weight_scale_2" in weight_name or "input_scale" in weight_name:
|
||||
local_expert_ids = self._map_global_expert_id(expert_id)
|
||||
if not local_expert_ids:
|
||||
return False if return_success else None
|
||||
value = loaded_weight.reshape(()).to(torch.float32)
|
||||
for local_expert_id in local_expert_ids:
|
||||
if shard_id in ("w1", "w3"):
|
||||
if "w13_" not in weight_name:
|
||||
return False if return_success else None
|
||||
param.data[local_expert_id, 0 if shard_id == "w1" else 1] = value
|
||||
elif shard_id == "w2":
|
||||
if "w2_" not in weight_name:
|
||||
return False if return_success else None
|
||||
param.data[local_expert_id] = value
|
||||
else:
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
return True if return_success else None
|
||||
return super().weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success,
|
||||
)
|
||||
|
||||
def finalize_weights(self, shared_experts: DeepseekV4MLP | None = None) -> None:
|
||||
# The FlashInfer megakernel has no shared-expert fusion; the caller's
|
||||
# serial shared MLP path handles shared_experts.
|
||||
if self._mega_layer is not None:
|
||||
return
|
||||
if self.w13_weight is None:
|
||||
return
|
||||
|
||||
self._check_runtime_supported()
|
||||
ensure_fi_moe_ep_runtime(self._vllm_config)
|
||||
|
||||
if self._nvfp4_prequant:
|
||||
# NVFP4 checkpoint: hand the packed weights + both scale
|
||||
# planes straight to the backend (no dequant->requant);
|
||||
# per-expert globals become fc1/fc2 epilogue alphas staged
|
||||
# at every forward via MoEEpTensors.
|
||||
weights, fc1_alpha, fc2_alpha = nvfp4_prequant_pack_and_alphas(
|
||||
self.w13_weight.data,
|
||||
self.w13_weight_scale.data,
|
||||
self.w13_weight_scale_2.data,
|
||||
self.w2_weight.data,
|
||||
self.w2_weight_scale.data,
|
||||
self.w2_weight_scale_2.data,
|
||||
intermediate_size=self.intermediate_size,
|
||||
)
|
||||
self._epilogue_alphas = (fc1_alpha, fc2_alpha)
|
||||
else:
|
||||
weights = mega_moe_weight_pack_from_params(
|
||||
self.w13_weight,
|
||||
self.w13_weight_scale,
|
||||
self.w2_weight,
|
||||
self.w2_weight_scale,
|
||||
megakernel=fi_moe_ep_backend_spec(
|
||||
self._vllm_config.kernel_config.moe_backend
|
||||
).megakernel,
|
||||
)
|
||||
self._mega_layer = build_fi_mega_layer(
|
||||
make_fi_moe_ep_bootstrap(),
|
||||
vllm_config=self._vllm_config,
|
||||
num_experts=self.num_experts,
|
||||
max_tokens_per_rank=self.max_num_tokens,
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=self.intermediate_size,
|
||||
top_k=self.top_k,
|
||||
activation_clamp=self._activation_clamp,
|
||||
weights=weights,
|
||||
)
|
||||
del weights
|
||||
# Allocate (or attach to) the pooled workspace before first
|
||||
# forward so warmup/capture never hits the lazy path.
|
||||
self._mega_layer._ensure_workspace()
|
||||
self.w13_weight = None
|
||||
self.w13_weight_scale = None
|
||||
self.w2_weight = None
|
||||
self.w2_weight_scale = None
|
||||
if self._nvfp4_prequant:
|
||||
self.w13_weight_scale_2 = None
|
||||
self.w2_weight_scale_2 = None
|
||||
self.w13_input_scale = None
|
||||
self.w2_input_scale = None
|
||||
|
||||
# EPLB is rejected for these backends in validate_fi_moe_ep_config, so
|
||||
# nothing constructs an EplbState and none of these run. They are kept
|
||||
# as explicit errors rather than deleted because inheriting the native
|
||||
# implementations would be worse: set_eplb_state would record a map
|
||||
# this forward path never consults, silently splitting routing from
|
||||
# weights, and get_expert_weights reads transformed-weight attributes
|
||||
# the flashinfer path releases after preprocess.
|
||||
_EPLB_UNSUPPORTED = (
|
||||
"EPLB is not supported with the flashinfer moe_ep backends; "
|
||||
"validate_fi_moe_ep_config should have rejected this "
|
||||
"configuration at startup."
|
||||
)
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
moe_layer_idx: int,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
raise NotImplementedError(self._EPLB_UNSUPPORTED)
|
||||
|
||||
def get_expert_weights(self) -> list[torch.Tensor]:
|
||||
raise NotImplementedError(self._EPLB_UNSUPPORTED)
|
||||
|
||||
def update_expert_map(self) -> None:
|
||||
raise NotImplementedError(self._EPLB_UNSUPPORTED)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
*,
|
||||
activation_clamp: float | None,
|
||||
fast_math: bool = True,
|
||||
) -> torch.Tensor:
|
||||
# fast_math is a native deep_gemm knob; the FI kernels have no
|
||||
# equivalent toggle, so it is accepted for signature parity only.
|
||||
if hidden_states.shape[0] > self.max_num_tokens:
|
||||
raise ValueError(
|
||||
f"DeepSeek V4 MegaMoE got {hidden_states.shape[0]} tokens, "
|
||||
f"but the symmetric buffer was sized for {self.max_num_tokens}."
|
||||
)
|
||||
|
||||
from flashinfer.moe_ep import MoEEpTensors
|
||||
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_padding = resolve_mega_moe_is_padding(num_tokens)
|
||||
topk_ids = apply_mega_moe_routing_preprocess(
|
||||
topk_ids,
|
||||
is_padding=is_padding,
|
||||
)
|
||||
|
||||
# Fast path: after the first successful full forward the layer is
|
||||
# immutable, so skip MoEEpMegaLayer.forward()'s per-call validation
|
||||
# and go straight to the kernel backend's stage_inputs + compute.
|
||||
alphas = self._epilogue_alphas
|
||||
fc1_alpha = alphas[0] if alphas is not None else None
|
||||
fc2_alpha = alphas[1] if alphas is not None else None
|
||||
|
||||
fast = self._fast_ctx
|
||||
if fast is not None:
|
||||
kernel, workspace, transformed, hidden_size, zero_copy = fast
|
||||
t = MoEEpTensors(
|
||||
hidden_states=hidden_states,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
fc1_alpha=fc1_alpha,
|
||||
fc2_alpha=fc2_alpha,
|
||||
)
|
||||
kernel.stage_inputs(t, workspace, quantize_input=True)
|
||||
if zero_copy:
|
||||
# Zero-copy (cutedsl backends): consume the workspace [:n]
|
||||
# view directly — valid under stream ordering until the next
|
||||
# MoE layer's launch on the shared workspace.
|
||||
return kernel.compute(workspace, transformed, output=None)
|
||||
# deep_gemm_mega's compute() requires a real output tensor.
|
||||
out = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
return kernel.compute(workspace, transformed, output=out)
|
||||
|
||||
ensure_fi_moe_ep_runtime(self._vllm_config)
|
||||
self.finalize_weights()
|
||||
assert self._mega_layer is not None
|
||||
|
||||
y = self._mega_layer.forward(
|
||||
MoEEpTensors(
|
||||
hidden_states=hidden_states,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
fc1_alpha=fc1_alpha,
|
||||
fc2_alpha=fc2_alpha,
|
||||
)
|
||||
)
|
||||
layer = self._mega_layer
|
||||
if hidden_states.dtype == torch.bfloat16:
|
||||
self._fast_ctx = (
|
||||
layer._kernel,
|
||||
layer._ensure_workspace(),
|
||||
layer._transformed,
|
||||
layer._fleet_params.token_hidden_size,
|
||||
# zero-copy output views are a cutedsl-backend contract
|
||||
layer._kernel.kernel_name() != "deep_gemm_mega",
|
||||
)
|
||||
return y
|
||||
|
||||
|
||||
DeepseekV4MegaMoEExpertsFI.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
@@ -11,7 +11,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
@@ -85,10 +85,6 @@ from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
is_fi_moe_ep_backend,
|
||||
validate_fi_moe_ep_config,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
@@ -807,10 +803,11 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.reduce_results = reduce_results
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
validate_fi_moe_ep_config(vllm_config)
|
||||
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_fi_mega_moe = is_fi_moe_ep_backend(moe_backend)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
self.use_native_mega_moe = moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
if (
|
||||
self.use_native_mega_moe
|
||||
and not vllm_config.parallel_config.enable_expert_parallel
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
"Enable it with --enable-expert-parallel, or pick a different "
|
||||
@@ -832,15 +829,15 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.swiglu_limit = config.swiglu_limit
|
||||
self.renormalize = config.norm_topk_prob
|
||||
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
|
||||
if self.use_mega_moe and self.scoring_func != "sqrtsoftplus":
|
||||
if self.use_native_mega_moe and self.scoring_func != "sqrtsoftplus":
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only."
|
||||
)
|
||||
if self.use_mega_moe and not self.renormalize:
|
||||
if self.use_native_mega_moe and not self.renormalize:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE requires normalized top-k probabilities."
|
||||
)
|
||||
if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
|
||||
if self.use_native_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype="
|
||||
f"{config.expert_dtype!r}. Drop --kernel-config moe_backend="
|
||||
@@ -864,7 +861,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
image_sentinel_lo if getattr(config, "vision_n_layers", 0) > 0 else 0
|
||||
)
|
||||
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
|
||||
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
|
||||
self.hash_indices_dtype = (
|
||||
torch.int64 if self.use_native_mega_moe else torch.int32
|
||||
)
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
# Use randint instead of empty to avoid garbage values causing
|
||||
@@ -908,12 +907,12 @@ class DeepseekV4MoE(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=self.use_mega_moe,
|
||||
reduce_results=self.use_native_mega_moe,
|
||||
is_sequence_parallel=use_sequence_parallel,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
self._init_mega_moe_experts(vllm_config, config, prefix)
|
||||
else:
|
||||
self._init_fused_moe_experts(vllm_config, config, quant_config, prefix)
|
||||
@@ -950,28 +949,13 @@ class DeepseekV4MoE(nn.Module):
|
||||
# Native DeepGEMM fusion requires each EP rank to own the complete
|
||||
# shared MLP. Sequence parallel replicates those weights while sharding
|
||||
# tokens. TP=1 is also naturally replicated. With PP+TP the shared MLP
|
||||
# remains tensor-sharded, so retain the serial path. The FlashInfer
|
||||
# megakernel has no shared-expert fusion, so it keeps the serial path.
|
||||
# remains tensor-sharded, so retain the serial path.
|
||||
fuse_shared_experts = bool(
|
||||
self.shared_experts is not None
|
||||
and not envs.VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION
|
||||
and (self.use_sequence_parallel or self.tp_size == 1)
|
||||
and not self.use_fi_mega_moe
|
||||
)
|
||||
|
||||
activation_clamp = (
|
||||
float(self.swiglu_limit) if self.swiglu_limit is not None else None
|
||||
)
|
||||
if self.use_fi_mega_moe:
|
||||
# Deferred: fi_moe subclasses DeepseekV4MegaMoEExperts, so a
|
||||
# module-level import here would be circular.
|
||||
from vllm.models.deepseek_v4.nvidia.fi_moe import (
|
||||
DeepseekV4MegaMoEExpertsFI,
|
||||
)
|
||||
|
||||
experts_cls: type[DeepseekV4MegaMoEExperts] = DeepseekV4MegaMoEExpertsFI
|
||||
else:
|
||||
experts_cls = DeepseekV4MegaMoEExperts
|
||||
expert_kwargs: dict[str, typing.Any] = dict(
|
||||
num_experts=self.n_physical_experts,
|
||||
num_local_experts=self.n_local_physical_experts,
|
||||
@@ -983,9 +967,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
if self.use_fi_mega_moe:
|
||||
expert_kwargs["activation_clamp"] = activation_clamp
|
||||
self.experts = experts_cls(vllm_config, **expert_kwargs)
|
||||
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
|
||||
|
||||
def _init_fused_moe_experts(
|
||||
self,
|
||||
@@ -1048,7 +1030,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
if bias_vl is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
|
||||
|
||||
if not self.use_mega_moe:
|
||||
if not self.use_native_mega_moe:
|
||||
return self._forward_fused_moe(hidden_states, input_ids)
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
@@ -1139,7 +1121,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
self.experts.finalize_weights(self.shared_experts)
|
||||
|
||||
|
||||
@@ -1179,12 +1161,15 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
moe_needs_token_sharded_input = (
|
||||
moe_backend in MEGA_MOE_BACKENDS or parallel_config.data_parallel_size > 1
|
||||
)
|
||||
return (
|
||||
parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.enable_expert_parallel
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and (use_mega_moe or parallel_config.data_parallel_size > 1)
|
||||
and moe_needs_token_sharded_input
|
||||
)
|
||||
|
||||
|
||||
@@ -1425,9 +1410,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
if (
|
||||
self.use_native_mega_moe
|
||||
and not vllm_config.parallel_config.enable_expert_parallel
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
"Enable it with --enable-expert-parallel, or pick a different "
|
||||
@@ -1564,7 +1554,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
@@ -1578,7 +1568,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
input_ids = sp_shard(input_ids)
|
||||
|
||||
mega_gate_metadata = None
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
input_ids,
|
||||
has_hash_routing=self.has_local_hash_moe,
|
||||
@@ -1802,7 +1792,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
|
||||
if first_layer.ffn.use_mega_moe:
|
||||
if first_layer.ffn.use_native_mega_moe:
|
||||
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
|
||||
@@ -20,7 +20,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -95,7 +95,9 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
|
||||
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -186,7 +188,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
|
||||
inputs_embeds
|
||||
).unsqueeze(-2)
|
||||
mega_gate_metadata = None
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
routing_input_ids = input_ids
|
||||
if self.mtp_block.use_sequence_parallel:
|
||||
routing_input_ids = sp_shard(routing_input_ids)
|
||||
@@ -384,7 +386,7 @@ class DeepSeekV4MTP(nn.Module):
|
||||
|
||||
# Pre-compute expert mapping ONCE.
|
||||
first_layer = next(iter(self.model.layers.values()))
|
||||
if first_layer.mtp_block.ffn.use_mega_moe:
|
||||
if first_layer.mtp_block.ffn.use_native_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(
|
||||
self.config.n_routed_experts
|
||||
)
|
||||
|
||||
@@ -18,7 +18,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -81,7 +81,9 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
self.rms_norm_eps = config.rms_norm_eps
|
||||
self.num_hidden_layers = config.num_hidden_layers
|
||||
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.num_dspark_layers = (
|
||||
@@ -214,7 +216,7 @@ class DSparkDeepseekV4Model(nn.Module):
|
||||
inputs_embeds = sp_shard(inputs_embeds)
|
||||
input_ids = sp_shard(input_ids)
|
||||
mega_gate_metadata = None
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
input_ids,
|
||||
has_hash_routing=False,
|
||||
@@ -371,7 +373,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
are skipped here. ``embed_tokens``/``lm_head`` are aliased from the target.
|
||||
"""
|
||||
first_layer = self.model.layers[0]
|
||||
use_mega_moe = first_layer.ffn.use_mega_moe
|
||||
use_native_mega_moe = first_layer.ffn.use_native_mega_moe
|
||||
# Draft MoE layers use the dspark_* expert counts, not the
|
||||
# backbone's (see DeepseekV4MoE and the reference
|
||||
# ModelArgs.get_moe_config).
|
||||
@@ -379,7 +381,7 @@ class DSparkDeepseekV4ForCausalLM(nn.Module):
|
||||
getattr(self.config, "dspark_n_routed_experts", 0)
|
||||
or self.config.n_routed_experts
|
||||
)
|
||||
if use_mega_moe:
|
||||
if use_native_mega_moe:
|
||||
expert_mapping = make_deepseek_v4_expert_params_mapping(n_draft_experts)
|
||||
else:
|
||||
expert_mapping = fused_moe_make_expert_params_mapping(
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_engram_dp_size,
|
||||
get_pp_group,
|
||||
@@ -174,7 +174,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
|
||||
def defers_finalize(self, num_tokens: int) -> bool:
|
||||
return (
|
||||
not self.use_mega_moe
|
||||
not self.use_native_mega_moe
|
||||
and self.experts.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
)
|
||||
|
||||
@@ -244,12 +244,14 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
moe_needs_token_sharded_input = (
|
||||
vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
)
|
||||
return (
|
||||
parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.enable_expert_parallel
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and (use_mega_moe or parallel_config.data_parallel_size > 1)
|
||||
and (moe_needs_token_sharded_input or parallel_config.data_parallel_size > 1)
|
||||
)
|
||||
|
||||
|
||||
@@ -631,9 +633,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
if (
|
||||
self.use_native_mega_moe
|
||||
and not vllm_config.parallel_config.enable_expert_parallel
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
"Enable it with --enable-expert-parallel, or pick a different "
|
||||
@@ -865,7 +872,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
# Engram n-gram hashes for the whole (flattened) batch, computed once
|
||||
@@ -1015,7 +1022,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
def _mega_gate_metadata(
|
||||
self, input_ids: torch.Tensor | None
|
||||
) -> MegaGateRoutingMetadata | None:
|
||||
if not self.use_mega_moe:
|
||||
if not self.use_native_mega_moe:
|
||||
return None
|
||||
assert input_ids is not None
|
||||
return prepare_mega_gate_routing_metadata(
|
||||
@@ -1331,7 +1338,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
|
||||
if first_layer.ffn.use_mega_moe:
|
||||
if first_layer.ffn.use_native_mega_moe:
|
||||
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
|
||||
Reference in New Issue
Block a user