[DSv4.1] Integrate Mega-Gate from DeepGEMM (#56266)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Thien Tran
2026-09-17 12:06:36 -07:00
committed by GitHub
co-authored by OpenAI Codex zjy0516
parent a9a7e45f31
commit 41f9104fa6
9 changed files with 369 additions and 35 deletions
+14
View File
@@ -17,6 +17,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -51,12 +52,14 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
logger = init_logger(__name__)
@@ -96,6 +99,7 @@ class DSparkDeepseekV4Model(nn.Module):
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = getattr(config, "n_mtp_layers", None) or 3
@@ -235,6 +239,15 @@ class DSparkDeepseekV4Model(nn.Module):
)
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
@@ -247,6 +260,7 @@ class DSparkDeepseekV4Model(nn.Module):
post_mix,
res_mix,
residual,
mega_gate_metadata=mega_gate_metadata,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.use_sequence_parallel:
+110 -19
View File
@@ -98,6 +98,37 @@ from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
logger = init_logger(__name__)
class MegaGateRoutingMetadata(typing.NamedTuple):
safe_hash_input_ids: torch.Tensor | None
image_token_mask: torch.Tensor | None
hash_token_mask: torch.Tensor | None
def prepare_mega_gate_routing_metadata(
input_ids: torch.Tensor,
*,
has_hash_routing: bool,
image_sentinel_base_id: int | None,
) -> MegaGateRoutingMetadata:
if image_sentinel_base_id is None:
return MegaGateRoutingMetadata(
input_ids if has_hash_routing else None,
None,
torch.ones_like(input_ids, dtype=torch.bool) if has_hash_routing else None,
)
image_token_mask = (input_ids >= image_sentinel_base_id) & (
input_ids < image_sentinel_base_id + 5
)
if not has_hash_routing:
return MegaGateRoutingMetadata(None, image_token_mask, None)
return MegaGateRoutingMetadata(
torch.where(image_token_mask, 0, input_ids),
image_token_mask,
~image_token_mask,
)
class DeepseekV4MLP(nn.Module):
def __init__(
self,
@@ -803,6 +834,10 @@ class DeepseekV4MoE(nn.Module):
raise NotImplementedError(
"DeepSeek V4 MegaMoE currently supports sqrtsoftplus routing only."
)
if self.use_mega_moe and not self.renormalize:
raise NotImplementedError(
"DeepSeek V4 MegaMoE requires normalized top-k probabilities."
)
if self.use_mega_moe and getattr(config, "expert_dtype", "fp4") != "fp4":
raise NotImplementedError(
"DeepSeek V4 MegaMoE only supports fp4 experts; got expert_dtype="
@@ -1000,7 +1035,10 @@ class DeepseekV4MoE(nn.Module):
)
def forward(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None = None
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor | None = None,
mega_gate_metadata: MegaGateRoutingMetadata | None = None,
) -> torch.Tensor:
if self.gate.tid2eid is not None and input_ids is None:
raise ValueError("DeepSeek V4 hash MoE routing requires input_ids.")
@@ -1013,23 +1051,61 @@ class DeepseekV4MoE(nn.Module):
return self._forward_fused_moe(hidden_states, input_ids)
org_shape = hidden_states.shape
router_logits, _ = self.gate(hidden_states)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
scoring_func=self.scoring_func,
e_score_correction_bias=self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
else None,
topk=self.n_activated_experts,
renormalize=self.renormalize,
indices_type=self.hash_indices_dtype,
input_tokens=input_ids,
hash_indices_table=self.gate.tid2eid,
routed_scaling_factor=self.routed_scaling_factor,
bias_vl=bias_vl.data if bias_vl is not None else None,
image_sentinel_lo=self.image_sentinel_lo if bias_vl is not None else 0,
)
# Small local padded batches favor GateLinear; 128-expert gates cross earlier.
gate_threshold = 1 if self.gate.weight.shape[0] == 128 else 16
if hidden_states.shape[0] <= gate_threshold:
router_logits, _ = self.gate(hidden_states)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=router_logits,
scoring_func=self.scoring_func,
e_score_correction_bias=self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
else None,
topk=self.n_activated_experts,
renormalize=self.renormalize,
indices_type=self.hash_indices_dtype,
input_tokens=input_ids,
hash_indices_table=self.gate.tid2eid,
routed_scaling_factor=self.routed_scaling_factor,
bias_vl=bias_vl.data if bias_vl is not None else None,
image_sentinel_lo=self.image_sentinel_lo if bias_vl is not None else 0,
)
else:
from vllm.utils.deep_gemm import bf16_mega_gate
if mega_gate_metadata is None:
raise RuntimeError(
"MegaMoE routing metadata must be prepared once by the model."
)
unmapped_topk_idx = None
fix_routing_mask = None
if self.gate.tid2eid is not None:
if mega_gate_metadata.safe_hash_input_ids is None:
raise RuntimeError("Hash routing requires prepared input IDs.")
if mega_gate_metadata.hash_token_mask is None:
raise RuntimeError("Hash routing requires a prepared routing mask.")
unmapped_topk_idx = self.gate.tid2eid[
mega_gate_metadata.safe_hash_input_ids
]
fix_routing_mask = mega_gate_metadata.hash_token_mask
topk_weights, topk_ids = bf16_mega_gate(
hidden_states,
self.gate.weight,
self.n_activated_experts,
scoring_func=self.scoring_func,
routed_scaling_factor=self.routed_scaling_factor,
ep_rank=self.ep_rank,
bias=self.gate.e_score_correction_bias.data
if self.gate.e_score_correction_bias is not None
else None,
image_bias=bias_vl.data if bias_vl is not None else None,
image_token_mask=mega_gate_metadata.image_token_mask,
fix_routing_mask=fix_routing_mask,
unmapped_topk_idx=unmapped_topk_idx,
)
activation_clamp = (
float(self.swiglu_limit) if self.swiglu_limit is not None else None
)
@@ -1248,6 +1324,7 @@ class DeepseekV4DecoderLayer(nn.Module):
post_mix: torch.Tensor | None = None,
res_mix: torch.Tensor | None = None,
residual: torch.Tensor | None = None,
mega_gate_metadata: MegaGateRoutingMetadata | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
attn_norm_weight = self.attn_norm.weight.data
attn_norm_eps = self.attn_norm.variance_epsilon
@@ -1332,7 +1409,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_eps=ffn_norm_eps,
)
x = self.ffn(x, input_ids)
x = self.ffn(x, input_ids, mega_gate_metadata)
return x, residual, post_mix, res_mix
@@ -1394,6 +1471,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
self.has_local_hash_moe = self.start_layer < min(
self.end_layer, config.num_hash_layers
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, self.rms_norm_eps)
@@ -1496,6 +1576,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
hidden_states = sp_shard(hidden_states)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=self.has_local_hash_moe,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
residual, post_mix, res_mix = None, None, None
remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)
aux_hidden_states: list[torch.Tensor] = []
@@ -1511,6 +1601,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
post_mix,
res_mix,
residual,
mega_gate_metadata=mega_gate_metadata,
)
if idx + 1 in self.aux_hidden_state_layers:
# Reconstruct the aux hidden state for draft models
+20 -1
View File
@@ -20,6 +20,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -51,6 +52,7 @@ from vllm.models.common.ops.sequence_parallel import (
sp_padding_mask,
sp_shard,
)
from vllm.models.deepseek_v4.common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from vllm.models.deepseek_v4.common.ops.fused_mtp_input_rmsnorm import (
_FUSED_MTP_INPUT_RMSNORM_KERNEL,
_MTP_SHARED_HEAD_RMSNORM_KERNEL,
@@ -62,6 +64,7 @@ from .model import (
DeepseekV4Model,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
logger = init_logger(__name__)
@@ -92,6 +95,7 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
self.config = config
quant_config = vllm_config.quant_config
self.rms_norm_eps = config.rms_norm_eps
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -181,8 +185,23 @@ class DeepSeekV4MultiTokenPredictorLayer(nn.Module):
hidden_states = self.h_proj(previous_hidden_states) + self.e_proj(
inputs_embeds
).unsqueeze(-2)
mega_gate_metadata = None
if self.use_mega_moe:
routing_input_ids = input_ids
if self.mtp_block.use_sequence_parallel:
routing_input_ids = sp_shard(routing_input_ids)
mega_gate_metadata = prepare_mega_gate_routing_metadata(
routing_input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
hidden_states, residual, post_mix, res_mix = self.mtp_block(
positions=positions, x=hidden_states, input_ids=input_ids
positions=positions,
x=hidden_states,
input_ids=input_ids,
mega_gate_metadata=mega_gate_metadata,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
if self.mtp_block.use_sequence_parallel:
+14
View File
@@ -18,6 +18,7 @@ import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -50,12 +51,14 @@ from vllm.models.common.ops.sequence_parallel import (
sp_shard,
)
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID
from .model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
_linear_scale_param_name,
_use_sequence_parallel,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
logger = init_logger(__name__)
@@ -77,6 +80,7 @@ class DSparkDeepseekV4Model(nn.Module):
self.rms_norm_eps = config.rms_norm_eps
self.num_hidden_layers = config.num_hidden_layers
self.target_layer_ids = tuple(config.dspark_target_layer_ids)
self.use_mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.num_dspark_layers = (
@@ -202,6 +206,15 @@ class DSparkDeepseekV4Model(nn.Module):
)
inputs_embeds = sp_shard(inputs_embeds)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
# Expand to hc_mult copies for hyper-connections ([T, H] -> [T, hc, H]).
hidden_states = inputs_embeds.unsqueeze(-2).repeat(1, self.hc_mult, 1)
@@ -215,6 +228,7 @@ class DSparkDeepseekV4Model(nn.Module):
post_mix,
res_mix,
residual,
mega_gate_metadata=mega_gate_metadata,
)
hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix)
# Collapse the hc copies with the pre-mix from the last layer's FFN
+15 -1
View File
@@ -61,7 +61,9 @@ from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4MoE as DeepseekV4MoEBase,
)
from vllm.models.deepseek_v4.nvidia.model import (
MegaGateRoutingMetadata,
make_deepseek_v4_expert_params_mapping,
prepare_mega_gate_routing_metadata,
)
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import (
@@ -339,6 +341,7 @@ class DeepseekV4DecoderLayer(nn.Module):
engram_mask: torch.Tensor | None = None,
*,
capture_previous_aux: bool = False,
mega_gate_metadata: MegaGateRoutingMetadata | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor,
@@ -462,7 +465,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
)
x = self.ffn(x, input_ids)
x = self.ffn(x, input_ids, mega_gate_metadata)
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
@@ -705,6 +708,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
hidden_states = sp_shard(hidden_states)
input_ids = sp_shard(input_ids)
mega_gate_metadata = None
if self.use_mega_moe:
mega_gate_metadata = prepare_mega_gate_routing_metadata(
input_ids,
has_hash_routing=False,
image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID
if getattr(self.config, "vision_n_layers", 0) > 0
else None,
)
residual, post_mix, res_mix = None, None, None
pre_mix: torch.Tensor | None = None
if not get_pp_group().is_first_rank:
@@ -728,6 +741,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
engram_hashes,
engram_mask,
capture_previous_aux=idx in self.aux_hidden_state_layers,
mega_gate_metadata=mega_gate_metadata,
)
if previous_aux is not None:
# idx is the one-based id of the layer whose post this is.
+54
View File
@@ -164,6 +164,7 @@ _get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_mega_mhc_impl: Callable[..., Any] | None = None
_bf16_mega_gate_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
@@ -240,6 +241,7 @@ def _lazy_init() -> None:
global _get_sparse_mqa_logits_metadata_impl
global _get_paged_sparse_mqa_logits_metadata_impl
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
global _bf16_mega_gate_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
@@ -260,6 +262,7 @@ def _lazy_init() -> None:
or _get_paged_mqa_logits_metadata_impl is not None
or _tf32_hc_prenorm_gemm_impl is not None
or _mega_mhc_impl is not None
or _bf16_mega_gate_impl is not None
or _get_mk_alignment_for_contiguous_layout_impl is not None
or _transform_sf_into_required_layout_impl is not None
or _pack_ue8m0_to_int_impl is not None
@@ -315,6 +318,7 @@ def _lazy_init() -> None:
)
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
_bf16_mega_gate_impl = getattr(_dg, "bf16_mega_gate", None)
_get_mn_major_tma_aligned_tensor_impl = getattr(
_dg, "get_mn_major_tma_aligned_tensor", None
)
@@ -482,6 +486,55 @@ def cublaslt_gemm_nt(*args, **kwargs):
return _cublaslt_gemm_nt_impl(*args, **kwargs)
def bf16_mega_gate(
x: torch.Tensor,
weight: torch.Tensor,
num_topk: int,
*,
scoring_func: str,
routed_scaling_factor: float,
ep_rank: int,
bias: torch.Tensor | None = None,
image_bias: torch.Tensor | None = None,
image_token_mask: torch.Tensor | None = None,
fix_routing_mask: torch.Tensor | None = None,
unmapped_topk_idx: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run DeepGEMM Mega Gate.
The output order follows vLLM's router contract: weights, then indices.
Passing PyTorch-owned outputs to DeepGEMM makes their storage part of CUDA
graph capture.
"""
_lazy_init()
if _bf16_mega_gate_impl is None:
raise RuntimeError(
"DeepGEMM MegaMoE requires a DeepGEMM build with bf16_mega_gate."
)
topk_idx = torch.empty((x.shape[0], num_topk), dtype=torch.int64, device=x.device)
topk_weights = torch.empty(
(x.shape[0], num_topk), dtype=torch.float32, device=x.device
)
_bf16_mega_gate_impl(
x,
weight,
num_topk,
use_shared_as_routed=False,
num_shared_experts=0,
routed_scaling_factor=routed_scaling_factor,
ep_rank=ep_rank,
scoring_func=scoring_func,
bias=bias,
image_bias=image_bias,
image_token_mask=image_token_mask,
fix_routing_mask=fix_routing_mask,
unmapped_topk_idx=unmapped_topk_idx,
out=(topk_idx, topk_weights),
)
return topk_weights, topk_idx
def fp8_gemm_nt(*args, **kwargs):
_lazy_init()
if _fp8_gemm_nt_impl is None:
@@ -988,6 +1041,7 @@ def should_use_deepgemm_for_fp8_linear(
__all__ = [
"calc_diff",
"bf16_mega_gate",
"DeepGemmQuantScaleFMT",
"fp8_gemm_nt",
"fp8_einsum",