[Perf][Kernel] Integrate Mega-mHC from DeepGEMM for DeepSeek V4.1 (reopen of #56255) (#56962)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Juntian Liu
2026-09-15 04:01:06 -07:00
committed by GitHub
co-authored by Yongye Zhu Claude Opus 5 Thien Tran
parent 5be911ee87
commit bb55077411
6 changed files with 316 additions and 41 deletions
+34 -38
View File
@@ -20,7 +20,6 @@ from vllm.distributed import (
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
@@ -79,6 +78,7 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
from .ops.mega_mhc import mhc_shifted_post_pre
if typing.TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
@@ -403,25 +403,23 @@ class DeepseekV4DecoderLayer(nn.Module):
else:
# The collapse already reads the post-mapped streams, so the mean
# aux consumers want comes out of the same kernel.
residual, post_mix, res_mix, x, attn_pre, aux = (
mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
)
residual, post_mix, res_mix, x, attn_pre, aux = mhc_shifted_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_attn_fn,
self.hc_attn_scale,
self.hc_attn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=pre_mix,
norm_weight=self.attn_norm.weight,
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
)
if capture_previous_aux:
previous_aux = aux
@@ -433,24 +431,22 @@ class DeepseekV4DecoderLayer(nn.Module):
if self.use_sequence_parallel:
x = sp_reduce_scatter(x)
residual, post_mix, res_mix, x, ffn_pre, _ = (
mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=attn_pre,
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
)
residual, post_mix, res_mix, x, ffn_pre, _ = mhc_shifted_post_pre(
x,
residual,
post_mix,
res_mix,
self.hc_ffn_fn,
self.hc_ffn_scale,
self.hc_ffn_base,
self.rms_norm_eps,
self.hc_eps,
self.hc_eps,
self.hc_post_alpha,
self.hc_sinkhorn_iters,
pre_mix=attn_pre,
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
)
x = self.ffn(x, input_ids)
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import torch
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_import_deep_gemm,
is_deep_gemm_supported,
mega_mhc,
)
@functools.cache
def is_mega_mhc_supported(hidden_size: int, hc_mult: int) -> bool:
"""Return whether DeepGEMM can run the DSV4.1 shifted mHC kernel."""
if not (
is_deep_gemm_supported()
and current_platform.is_device_capability_family(100)
and hidden_size > 0
and hidden_size % 1024 == 0
and hc_mult == 4
):
return False
deep_gemm = _import_deep_gemm()
return deep_gemm is not None and callable(getattr(deep_gemm, "mega_mhc", None))
def mhc_shifted_post_pre_deep_gemm(
x: torch.Tensor,
residual: torch.Tensor,
shifted_prev_mix: torch.Tensor,
post_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
mix_scales: torch.Tensor,
mix_bases: torch.Tensor,
hc_norm_eps: float,
hc_pre_eps: float,
hc_post_scale: float,
sinkhorn_eps: float,
num_sinkhorn_iters: int,
rmsnorm_weight: torch.Tensor,
rmsnorm_eps: float,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Run DSV4.1 shifted post, next pre, and BF16 RMSNorm with Mega mHC."""
num_tokens, hidden_size = x.shape
hc_mult = residual.shape[1]
new_residual = torch.empty_like(residual)
new_prev_mix = x.new_empty(num_tokens, hc_mult, 1, dtype=torch.float32)
new_post_mix = torch.empty_like(post_mix)
new_comb_res_mix = torch.empty_like(comb_res_mix)
y_bf16 = x.new_empty(num_tokens, hidden_size, dtype=torch.bfloat16)
mega_mhc(
x=x,
residual=residual,
shifted_prev_mix=shifted_prev_mix.unsqueeze(-1),
post_mix=post_mix,
comb_res_mix=comb_res_mix,
fn=fn,
mix_scales=mix_scales,
mix_bases=mix_bases,
hc_mult=hc_mult,
hc_norm_eps=hc_norm_eps,
hc_pre_eps=hc_pre_eps,
hc_post_scale=hc_post_scale,
sinkhorn_eps=sinkhorn_eps,
num_sinkhorn_iters=num_sinkhorn_iters,
rmsnorm_weight=rmsnorm_weight,
rmsnorm_eps=rmsnorm_eps,
rmsnorm_scale=1.0,
new_residual=new_residual,
new_prev_mix=new_prev_mix,
new_post_mix=new_post_mix,
new_comb_res_mix=new_comb_res_mix,
y_bf16=y_bf16,
)
return (
new_residual,
new_post_mix,
new_comb_res_mix,
y_bf16,
new_prev_mix.squeeze(-1),
)
def mhc_shifted_post_pre(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Use Mega-mHC for carried mixing, retaining TileLang for aux capture."""
if (
pre_mix is not None
and norm_weight is not None
and not capture_aux
and x.shape[0] <= 1 << 20
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
):
outputs = mhc_shifted_post_pre_deep_gemm(
x,
residual,
pre_mix,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_post_mult_value,
hc_sinkhorn_eps,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
return *outputs, x.new_empty(0, x.shape[1])
return mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
capture_aux=capture_aux,
)
+14 -1
View File
@@ -159,6 +159,7 @@ _fp8_fp4_mqa_logits_impl: Callable[..., Any] | None = None
_fp8_fp4_paged_mqa_logits_impl: Callable[..., Any] | None = None
_get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_mega_mhc_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
_get_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = None
_get_theoretical_mk_alignment_for_contiguous_layout_impl: Callable[..., Any] | None = (
@@ -231,7 +232,7 @@ def _lazy_init() -> None:
global _grouped_impl, _grouped_masked_impl, _grouped_fp4_impl
global _fp8_fp4_mqa_logits_impl, _fp8_fp4_paged_mqa_logits_impl
global _get_paged_mqa_logits_metadata_impl
global _tf32_hc_prenorm_gemm_impl
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
global _get_theoretical_mk_alignment_for_contiguous_layout_impl
@@ -251,6 +252,7 @@ def _lazy_init() -> None:
or _fp8_fp4_paged_mqa_logits_impl is not None
or _get_paged_mqa_logits_metadata_impl is not None
or _tf32_hc_prenorm_gemm_impl is not None
or _mega_mhc_impl is not None
or _get_mk_alignment_for_contiguous_layout_impl is not None
or _transform_sf_into_required_layout_impl is not None
or _pack_ue8m0_to_int_impl is not None
@@ -294,6 +296,7 @@ def _lazy_init() -> None:
_dg, "get_paged_mqa_logits_metadata", None
)
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
_get_mn_major_tma_aligned_tensor_impl = getattr(
_dg, "get_mn_major_tma_aligned_tensor", None
)
@@ -704,6 +707,15 @@ def tf32_hc_prenorm_gemm(
)
def mega_mhc(*args: Any, **kwargs: Any) -> None:
"""Run DeepGEMM Mega mHC with caller-owned output tensors."""
_lazy_init()
if _mega_mhc_impl is None:
_missing()
return
_mega_mhc_impl(*args, **kwargs)
def _ceil_to_ue8m0(x: torch.Tensor):
return torch.pow(2.0, torch.ceil(torch.log2(x.abs())))
@@ -797,6 +809,7 @@ __all__ = [
"per_block_cast_to_fp8",
"is_deep_gemm_e8m0_used",
"is_deep_gemm_supported",
"mega_mhc",
"get_num_sms",
"set_num_sms",
"should_use_deepgemm_for_fp8_linear",