[feat] FlashInfer CuteDSL MegaMoE integration (#54049)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -11,7 +11,7 @@ import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS, NATIVE_MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_ep_group,
|
||||
get_pp_group,
|
||||
@@ -85,10 +85,6 @@ from vllm.models.deepseek_v4.nvidia.flashmla import DeepseekV4FlashMLAAttention
|
||||
from vllm.models.deepseek_v4.nvidia.ops.prepare_megamoe import prepare_megamoe_inputs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
is_fi_moe_ep_backend,
|
||||
validate_fi_moe_ep_config,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
@@ -807,10 +803,11 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.reduce_results = reduce_results
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
validate_fi_moe_ep_config(vllm_config)
|
||||
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_fi_mega_moe = is_fi_moe_ep_backend(moe_backend)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
self.use_native_mega_moe = moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
if (
|
||||
self.use_native_mega_moe
|
||||
and not vllm_config.parallel_config.enable_expert_parallel
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
"Enable it with --enable-expert-parallel, or pick a different "
|
||||
@@ -832,15 +829,15 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.swiglu_limit = config.swiglu_limit
|
||||
self.renormalize = config.norm_topk_prob
|
||||
self.scoring_func = getattr(config, "scoring_func", "sqrtsoftplus")
|
||||
if self.use_mega_moe and self.scoring_func != "sqrtsoftplus":
|
||||
if self.use_native_mega_moe and self.scoring_func != "sqrtsoftplus":
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only."
|
||||
)
|
||||
if self.use_mega_moe and not self.renormalize:
|
||||
if self.use_native_mega_moe and not self.renormalize:
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE requires normalized top-k probabilities."
|
||||
)
|
||||
if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
|
||||
if self.use_native_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype="
|
||||
f"{config.expert_dtype!r}. Drop --kernel-config moe_backend="
|
||||
@@ -864,7 +861,9 @@ class DeepseekV4MoE(nn.Module):
|
||||
image_sentinel_lo if getattr(config, "vision_n_layers", 0) > 0 else 0
|
||||
)
|
||||
is_hash_moe = extract_layer_index(prefix) < num_hash_layers
|
||||
self.hash_indices_dtype = torch.int64 if self.use_mega_moe else torch.int32
|
||||
self.hash_indices_dtype = (
|
||||
torch.int64 if self.use_native_mega_moe else torch.int32
|
||||
)
|
||||
if is_hash_moe:
|
||||
# hash MoE doesn't use e_score_correction_bias
|
||||
# Use randint instead of empty to avoid garbage values causing
|
||||
@@ -908,12 +907,12 @@ class DeepseekV4MoE(nn.Module):
|
||||
hidden_act=config.hidden_act,
|
||||
swiglu_limit=self.swiglu_limit,
|
||||
quant_config=quant_config,
|
||||
reduce_results=self.use_mega_moe,
|
||||
reduce_results=self.use_native_mega_moe,
|
||||
is_sequence_parallel=use_sequence_parallel,
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
)
|
||||
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
self._init_mega_moe_experts(vllm_config, config, prefix)
|
||||
else:
|
||||
self._init_fused_moe_experts(vllm_config, config, quant_config, prefix)
|
||||
@@ -950,28 +949,13 @@ class DeepseekV4MoE(nn.Module):
|
||||
# Native DeepGEMM fusion requires each EP rank to own the complete
|
||||
# shared MLP. Sequence parallel replicates those weights while sharding
|
||||
# tokens. TP=1 is also naturally replicated. With PP+TP the shared MLP
|
||||
# remains tensor-sharded, so retain the serial path. The FlashInfer
|
||||
# megakernel has no shared-expert fusion, so it keeps the serial path.
|
||||
# remains tensor-sharded, so retain the serial path.
|
||||
fuse_shared_experts = bool(
|
||||
self.shared_experts is not None
|
||||
and not envs.VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION
|
||||
and (self.use_sequence_parallel or self.tp_size == 1)
|
||||
and not self.use_fi_mega_moe
|
||||
)
|
||||
|
||||
activation_clamp = (
|
||||
float(self.swiglu_limit) if self.swiglu_limit is not None else None
|
||||
)
|
||||
if self.use_fi_mega_moe:
|
||||
# Deferred: fi_moe subclasses DeepseekV4MegaMoEExperts, so a
|
||||
# module-level import here would be circular.
|
||||
from vllm.models.deepseek_v4.nvidia.fi_moe import (
|
||||
DeepseekV4MegaMoEExpertsFI,
|
||||
)
|
||||
|
||||
experts_cls: type[DeepseekV4MegaMoEExperts] = DeepseekV4MegaMoEExpertsFI
|
||||
else:
|
||||
experts_cls = DeepseekV4MegaMoEExperts
|
||||
expert_kwargs: dict[str, typing.Any] = dict(
|
||||
num_experts=self.n_physical_experts,
|
||||
num_local_experts=self.n_local_physical_experts,
|
||||
@@ -983,9 +967,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
num_shared_experts=(self.n_shared_experts if fuse_shared_experts else 0),
|
||||
prefix=f"{prefix}.experts",
|
||||
)
|
||||
if self.use_fi_mega_moe:
|
||||
expert_kwargs["activation_clamp"] = activation_clamp
|
||||
self.experts = experts_cls(vllm_config, **expert_kwargs)
|
||||
self.experts = DeepseekV4MegaMoEExperts(vllm_config, **expert_kwargs)
|
||||
|
||||
def _init_fused_moe_experts(
|
||||
self,
|
||||
@@ -1048,7 +1030,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
if bias_vl is not None and input_ids is None:
|
||||
raise ValueError("DeepSeek V4 vision MoE routing requires input_ids.")
|
||||
|
||||
if not self.use_mega_moe:
|
||||
if not self.use_native_mega_moe:
|
||||
return self._forward_fused_moe(hidden_states, input_ids)
|
||||
|
||||
org_shape = hidden_states.shape
|
||||
@@ -1139,7 +1121,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
return final_hidden_states.view(org_shape)
|
||||
|
||||
def finalize_mega_moe_weights(self) -> None:
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
self.experts.finalize_weights(self.shared_experts)
|
||||
|
||||
|
||||
@@ -1179,12 +1161,15 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
|
||||
def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
|
||||
parallel_config = vllm_config.parallel_config
|
||||
use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
moe_needs_token_sharded_input = (
|
||||
moe_backend in MEGA_MOE_BACKENDS or parallel_config.data_parallel_size > 1
|
||||
)
|
||||
return (
|
||||
parallel_config.pipeline_parallel_size == 1
|
||||
and parallel_config.enable_expert_parallel
|
||||
and parallel_config.tensor_parallel_size > 1
|
||||
and (use_mega_moe or parallel_config.data_parallel_size > 1)
|
||||
and moe_needs_token_sharded_input
|
||||
)
|
||||
|
||||
|
||||
@@ -1425,9 +1410,14 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
|
||||
self.use_native_mega_moe = (
|
||||
vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS
|
||||
)
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
if self.use_mega_moe and not vllm_config.parallel_config.enable_expert_parallel:
|
||||
if (
|
||||
self.use_native_mega_moe
|
||||
and not vllm_config.parallel_config.enable_expert_parallel
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"DeepSeek V4 MegaMoE currently requires expert parallel. "
|
||||
"Enable it with --enable-expert-parallel, or pick a different "
|
||||
@@ -1564,7 +1554,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
input_ids = input_ids.to(torch.int64)
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
@@ -1578,7 +1568,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
input_ids = sp_shard(input_ids)
|
||||
|
||||
mega_gate_metadata = None
|
||||
if self.use_mega_moe:
|
||||
if self.use_native_mega_moe:
|
||||
mega_gate_metadata = prepare_mega_gate_routing_metadata(
|
||||
input_ids,
|
||||
has_hash_routing=self.has_local_hash_moe,
|
||||
@@ -1802,7 +1792,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
first_layer = next(iter(islice(self.layers, self.start_layer, self.end_layer)))
|
||||
if first_layer.ffn.use_mega_moe:
|
||||
if first_layer.ffn.use_native_mega_moe:
|
||||
return make_deepseek_v4_expert_params_mapping(self.config.n_routed_experts)
|
||||
# Params for weights, fp8 weight scales, fp8 activation scales
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
|
||||
Reference in New Issue
Block a user