[DSv4.1] Integrate Mega-Gate from DeepGEMM (#56266)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Thien Tran
2026-09-17 12:06:36 -07:00
committed by GitHub
co-authored by OpenAI Codex zjy0516
parent a9a7e45f31
commit 41f9104fa6
9 changed files with 369 additions and 35 deletions
+54
View File
@@ -164,6 +164,7 @@ _get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_mega_mhc_impl: Callable[..., Any] | None = None
_bf16_mega_gate_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
@@ -240,6 +241,7 @@ def _lazy_init() -> None:
global _get_sparse_mqa_logits_metadata_impl
global _get_paged_sparse_mqa_logits_metadata_impl
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
global _bf16_mega_gate_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
@@ -260,6 +262,7 @@ def _lazy_init() -> None:
or _get_paged_mqa_logits_metadata_impl is not None
or _tf32_hc_prenorm_gemm_impl is not None
or _mega_mhc_impl is not None
or _bf16_mega_gate_impl is not None
or _get_mk_alignment_for_contiguous_layout_impl is not None
or _transform_sf_into_required_layout_impl is not None
or _pack_ue8m0_to_int_impl is not None
@@ -315,6 +318,7 @@ def _lazy_init() -> None:
)
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
_bf16_mega_gate_impl = getattr(_dg, "bf16_mega_gate", None)
_get_mn_major_tma_aligned_tensor_impl = getattr(
_dg, "get_mn_major_tma_aligned_tensor", None
)
@@ -482,6 +486,55 @@ def cublaslt_gemm_nt(*args, **kwargs):
return _cublaslt_gemm_nt_impl(*args, **kwargs)
def bf16_mega_gate(
x: torch.Tensor,
weight: torch.Tensor,
num_topk: int,
*,
scoring_func: str,
routed_scaling_factor: float,
ep_rank: int,
bias: torch.Tensor | None = None,
image_bias: torch.Tensor | None = None,
image_token_mask: torch.Tensor | None = None,
fix_routing_mask: torch.Tensor | None = None,
unmapped_topk_idx: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run DeepGEMM Mega Gate.
The output order follows vLLM's router contract: weights, then indices.
Passing PyTorch-owned outputs to DeepGEMM makes their storage part of CUDA
graph capture.
"""
_lazy_init()
if _bf16_mega_gate_impl is None:
raise RuntimeError(
"DeepGEMM MegaMoE requires a DeepGEMM build with bf16_mega_gate."
)
topk_idx = torch.empty((x.shape[0], num_topk), dtype=torch.int64, device=x.device)
topk_weights = torch.empty(
(x.shape[0], num_topk), dtype=torch.float32, device=x.device
)
_bf16_mega_gate_impl(
x,
weight,
num_topk,
use_shared_as_routed=False,
num_shared_experts=0,
routed_scaling_factor=routed_scaling_factor,
ep_rank=ep_rank,
scoring_func=scoring_func,
bias=bias,
image_bias=image_bias,
image_token_mask=image_token_mask,
fix_routing_mask=fix_routing_mask,
unmapped_topk_idx=unmapped_topk_idx,
out=(topk_idx, topk_weights),
)
return topk_weights, topk_idx
def fp8_gemm_nt(*args, **kwargs):
_lazy_init()
if _fp8_gemm_nt_impl is None:
@@ -988,6 +1041,7 @@ def should_use_deepgemm_for_fp8_linear(
__all__ = [
"calc_diff",
"bf16_mega_gate",
"DeepGemmQuantScaleFMT",
"fp8_gemm_nt",
"fp8_einsum",