[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
+33 -43
View File
@@ -11,7 +11,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
from vllm.distributed import (
get_ep_group,
get_pp_group,
@@ -85,10 +85,6 @@ from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.flashinfer_moe_ep import (
is_fi_moe_ep_backend,
validate_fi_moe_ep_config,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
@@ -807,10 +803,11 @@ class DeepseekV4MoE(nn.Module):
self.use_sequence_parallel = use_sequence_parallel
self.reduce_results = reduce_results
moe_backend = vllm_config.kernel_config.moe_backend
validate_fi_moe_ep_config(vllm_config)
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
self.use_fi_mega_moe = is_fi_moe_ep_backend(moe_backend)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
self.use_native_mega_moe = moe_backend in NATIVE_MEGA_MOE_BACKENDS
if (
self.use_native_mega_moe
and not vllm_config.parallel_config.enable_expert_parallel
):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
"Enable it with --enable-expert-parallel, or pick a different "
@@ -832,15 +829,15 @@ class DeepseekV4MoE(nn.Module):
self.swiglu_limit = config.swiglu_limit
self.renormalize = config.norm_topk_prob
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
if self.use_mega_moe and self.scoring_func != "sqrtsoftplus":
if self.use_native_mega_moe and self.scoring_func != "sqrtsoftplus":
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only."
)
if self.use_mega_moe and not self.renormalize:
if self.use_native_mega_moe and not self.renormalize:
raise NotImplementedError(
"DeepSeek V4 MegaMoE requires normalized top-k probabilities."
)
if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
if self.use_native_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
raise NotImplementedError(
"DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype="
f"{config.expert_dtype!r}. Drop --kernel-config moe_backend="
@@ -864,7 +861,9 @@ class DeepseekV4MoE(nn.Module):
image_sentinel_lo if getattr(config, "vision_n_layers", 0) > 0 else 0
)
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
self.hash_indices_dtype = (
torch.int64 if self.use_native_mega_moe else torch.int32
)
if is_hash_moe:
# hash MoE doesn't use e_score_correction_bias
# Use randint instead of empty to avoid garbage values causing
@@ -908,12 +907,12 @@ class DeepseekV4MoE(nn.Module):
hidden_act=config.hidden_act,
swiglu_limit=self.swiglu_limit,
quant_config=quant_config,
reduce_results=self.use_mega_moe,
reduce_results=self.use_native_mega_moe,
is_sequence_parallel=use_sequence_parallel,
prefix=f"{prefix}.shared_experts",
)
if self.use_mega_moe:
if self.use_native_mega_moe:
self._init_mega_moe_experts(vllm_config, config, prefix)
else:
self._init_fused_moe_experts(vllm_config, config, quant_config, prefix)
@@ -950,28 +949,13 @@ class DeepseekV4MoE(nn.Module):
# Native DeepGEMM fusion requires each EP rank to own the complete
# shared MLP. Sequence parallel replicates those weights while sharding
# tokens. TP=1 is also naturally replicated. With PP+TP the shared MLP
# remains tensor-sharded, so retain the serial path. The FlashInfer
# megakernel has no shared-expert fusion, so it keeps the serial path.
# remains tensor-sharded, so retain the serial path.
fuse_shared_experts = bool(
self.shared_experts is not None
and not envs.VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION
and (self.use_sequence_parallel or self.tp_size == 1)
and not self.use_fi_mega_moe
)
activation_clamp = (
float(self.swiglu_limit) if self.swiglu_limit is not None else None
)
if self.use_fi_mega_moe:
# Deferred: fi_moe subclasses DeepseekV4MegaMoEExperts, so a
# module-level import here would be circular.
from vllm.models.deepseek_v4.nvidia.fi_moe import (
DeepseekV4MegaMoEExpertsFI,
)
experts_cls: type[DeepseekV4MegaMoEExperts] = DeepseekV4MegaMoEExpertsFI
else:
experts_cls = DeepseekV4MegaMoEExperts
expert_kwargs: dict[str, typing.Any] = dict(
num_experts=self.n_physical_experts,
num_local_experts=self.n_local_physical_experts,
@@ -983,9 +967,7 @@ class DeepseekV4MoE(nn.Module):
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
prefix=f"{prefix}.experts",
)
if self.use_fi_mega_moe:
expert_kwargs["activation_clamp"] = activation_clamp
self.experts = experts_cls(vllm_config, **expert_kwargs)
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
def _init_fused_moe_experts(
self,
@@ -1048,7 +1030,7 @@ class DeepseekV4MoE(nn.Module):
if bias_vl is not None and input_ids is None:
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
if not self.use_mega_moe:
if not self.use_native_mega_moe:
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
@@ -1139,7 +1121,7 @@ class DeepseekV4MoE(nn.Module):
return final_hidden_states.view(org_shape)
def finalize_mega_moe_weights(self) -> None:
if self.use_mega_moe:
if self.use_native_mega_moe:
self.experts.finalize_weights(self.shared_experts)
@@ -1179,12 +1161,15 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
parallel_config = vllm_config.parallel_config
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
moe_backend = vllm_config.kernel_config.moe_backend
moe_needs_token_sharded_input = (
moe_backend in MEGA_MOE_BACKENDS or parallel_config.data_parallel_size > 1
)
return (
parallel_config.pipeline_parallel_size == 1
and parallel_config.enable_expert_parallel
and parallel_config.tensor_parallel_size > 1
and (use_mega_moe or parallel_config.data_parallel_size > 1)
and moe_needs_token_sharded_input
)
@@ -1425,9 +1410,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.config = config
self.quant_config = quant_config
self.parallel_config = vllm_config.parallel_config
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_native_mega_moe = (
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
)
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
if (
self.use_native_mega_moe
and not vllm_config.parallel_config.enable_expert_parallel
):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently requires expert parallel. "
"Enable it with --enable-expert-parallel, or pick a different "
@@ -1564,7 +1554,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
if self.use_mega_moe:
if self.use_native_mega_moe:
input_ids = input_ids.to(torch.int64)
full_num_tokens = positions.shape[0]
@@ -1578,7 +1568,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
if self.use_native_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=self.has_local_hash_moe,
@@ -1802,7 +1792,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
if first_layer.ffn.use_mega_moe:
if first_layer.ffn.use_native_mega_moe:
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)