[DSv4.1] Integrate Mega-Gate from DeepGEMM (#56266)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
OpenAI Codex
zjy0516
parent
a9a7e45f31
commit
41f9104fa6
@@ -164,6 +164,7 @@ _get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
|
||||
_mega_mhc_impl: Callable[..., Any] | None = None
|
||||
_bf16_mega_gate_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
|
||||
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
|
||||
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
|
||||
@@ -240,6 +241,7 @@ def _lazy_init() -> None:
|
||||
global _get_sparse_mqa_logits_metadata_impl
|
||||
global _get_paged_sparse_mqa_logits_metadata_impl
|
||||
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
|
||||
global _bf16_mega_gate_impl
|
||||
global _get_mn_major_tma_aligned_tensor_impl
|
||||
global _get_mk_alignment_for_contiguous_layout_impl
|
||||
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
|
||||
@@ -260,6 +262,7 @@ def _lazy_init() -> None:
|
||||
or _get_paged_mqa_logits_metadata_impl is not None
|
||||
or _tf32_hc_prenorm_gemm_impl is not None
|
||||
or _mega_mhc_impl is not None
|
||||
or _bf16_mega_gate_impl is not None
|
||||
or _get_mk_alignment_for_contiguous_layout_impl is not None
|
||||
or _transform_sf_into_required_layout_impl is not None
|
||||
or _pack_ue8m0_to_int_impl is not None
|
||||
@@ -315,6 +318,7 @@ def _lazy_init() -> None:
|
||||
)
|
||||
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
|
||||
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
|
||||
_bf16_mega_gate_impl = getattr(_dg, "bf16_mega_gate", None)
|
||||
_get_mn_major_tma_aligned_tensor_impl = getattr(
|
||||
_dg, "get_mn_major_tma_aligned_tensor", None
|
||||
)
|
||||
@@ -482,6 +486,55 @@ def cublaslt_gemm_nt(*args, **kwargs):
|
||||
return _cublaslt_gemm_nt_impl(*args, **kwargs)
|
||||
|
||||
|
||||
def bf16_mega_gate(
|
||||
x: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
num_topk: int,
|
||||
*,
|
||||
scoring_func: str,
|
||||
routed_scaling_factor: float,
|
||||
ep_rank: int,
|
||||
bias: torch.Tensor | None = None,
|
||||
image_bias: torch.Tensor | None = None,
|
||||
image_token_mask: torch.Tensor | None = None,
|
||||
fix_routing_mask: torch.Tensor | None = None,
|
||||
unmapped_topk_idx: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run DeepGEMM Mega Gate.
|
||||
|
||||
The output order follows vLLM's router contract: weights, then indices.
|
||||
Passing PyTorch-owned outputs to DeepGEMM makes their storage part of CUDA
|
||||
graph capture.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _bf16_mega_gate_impl is None:
|
||||
raise RuntimeError(
|
||||
"DeepGEMM MegaMoE requires a DeepGEMM build with bf16_mega_gate."
|
||||
)
|
||||
|
||||
topk_idx = torch.empty((x.shape[0], num_topk), dtype=torch.int64, device=x.device)
|
||||
topk_weights = torch.empty(
|
||||
(x.shape[0], num_topk), dtype=torch.float32, device=x.device
|
||||
)
|
||||
_bf16_mega_gate_impl(
|
||||
x,
|
||||
weight,
|
||||
num_topk,
|
||||
use_shared_as_routed=False,
|
||||
num_shared_experts=0,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
ep_rank=ep_rank,
|
||||
scoring_func=scoring_func,
|
||||
bias=bias,
|
||||
image_bias=image_bias,
|
||||
image_token_mask=image_token_mask,
|
||||
fix_routing_mask=fix_routing_mask,
|
||||
unmapped_topk_idx=unmapped_topk_idx,
|
||||
out=(topk_idx, topk_weights),
|
||||
)
|
||||
return topk_weights, topk_idx
|
||||
|
||||
|
||||
def fp8_gemm_nt(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _fp8_gemm_nt_impl is None:
|
||||
@@ -988,6 +1041,7 @@ def should_use_deepgemm_for_fp8_linear(
|
||||
|
||||
__all__ = [
|
||||
"calc_diff",
|
||||
"bf16_mega_gate",
|
||||
"DeepGemmQuantScaleFMT",
|
||||
"fp8_gemm_nt",
|
||||
"fp8_einsum",
|
||||
|
||||
Reference in New Issue
Block a user