[Perf][Kernel] Integrate Mega-mHC from DeepGEMM for DeepSeek V4.1 (reopen of #56255) (#56962)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Juntian Liu
2026-09-15 04:01:06 -07:00
committed by GitHub
co-authored by Yongye Zhu Claude Opus 5 Thien Tran
parent 5be911ee87
commit bb55077411
6 changed files with 316 additions and 41 deletions
+6 -1
View File
@@ -27,6 +27,7 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
fp8_gemm_nt,
get_tma_aligned_size,
is_deep_gemm_e8m0_used,
per_block_cast_to_fp8,
should_use_deepgemm_for_fp8_linear,
)
@@ -291,7 +292,11 @@ def test_w8a8_block_fp8_deep_gemm_matmul(M, N, K, block_size, out_dtype, seed):
A_fp8, As_fp8 = per_token_group_quant_fp8(
A_fp32, block_size[1], column_major_scales=True, tma_aligned_scales=True
)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(B_fp32, block_size=block_size)
B_fp8, Bs_fp8 = per_block_cast_to_fp8(
B_fp32,
block_size=block_size,
use_ue8m0=is_deep_gemm_e8m0_used(),
)
As = As_fp8.to(torch.float32)
Bs = Bs_fp8.to(torch.float32)
+104 -1
View File
@@ -11,6 +11,7 @@ import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_torch_hc_prenorm_gemm,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
from vllm.model_executor.kernels.mhc.tilelang_kernels import (
@@ -39,6 +40,10 @@ from vllm.models.deepseek_v4.nvidia.model import (
from vllm.models.deepseek_v41.nvidia.model import (
DeepseekV4DecoderLayer as DeepseekV41DecoderLayer,
)
from vllm.models.deepseek_v41.nvidia.ops.mega_mhc import (
is_mega_mhc_supported,
mhc_shifted_post_pre_deep_gemm,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -495,7 +500,7 @@ def test_deepseek_v41_decoder_mixes_match_torch(
reference,
)
monkeypatch.setattr(
"vllm.models.deepseek_v41.nvidia.model.mhc_fused_post_pre_delayed_tilelang",
"vllm.models.deepseek_v41.nvidia.model.mhc_shifted_post_pre",
fused_reference,
)
expected = decoder(x, positions, None, **kwargs)
@@ -580,6 +585,104 @@ def test_mhc_pre_delayed_custom_op_supports_compile(carried):
)
@pytest.mark.skipif(not HAS_TILELANG_MHC, reason="TileLang MHC support required")
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="DeepGEMM Mega mHC requires SM100-family CUDA",
)
@pytest.mark.parametrize("num_tokens", [0, 1, 17, 128, 1024])
@pytest.mark.parametrize("hidden_size", [5120, 7168])
def test_deep_gemm_mega_mhc_correctness(num_tokens, hidden_size):
set_random_seed(0)
hc_mult = 4
if not is_mega_mhc_supported(hidden_size, 4):
pytest.skip("DeepGEMM Mega mHC is not available")
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=DEVICE)
residual = torch.randn(
num_tokens, hc_mult, hidden_size, dtype=torch.bfloat16, device=DEVICE
)
previous_mix = torch.rand(num_tokens, hc_mult, device=DEVICE)
post_mix = torch.rand(num_tokens, hc_mult, 1, device=DEVICE)
res_mix = torch.rand(num_tokens, hc_mult, hc_mult, device=DEVICE)
fn = torch.randn(24, hc_mult * hidden_size, device=DEVICE) * 0.01
scale = torch.randn(3, device=DEVICE) * 0.1
base = torch.randn(24, device=DEVICE) * 0.1
norm_weight = torch.empty(
hidden_size, dtype=torch.bfloat16, device=DEVICE
).uniform_(0.9, 1.1)
expected_residual = (
mhc_post_tilelang(x, residual, post_mix, res_mix) if num_tokens else residual
)
(
expected_post_mix,
expected_res_mix,
expected_y_bf16,
expected_previous_mix,
) = mhc_pre_delayed_tilelang(
expected_residual,
fn,
scale,
base,
2e-5,
3e-4,
2e-6,
1.25,
10,
pre_mix=previous_mix,
norm_weight=norm_weight,
norm_eps=7e-6,
)
args = (
x,
residual,
previous_mix,
post_mix,
res_mix,
fn,
scale,
base,
2e-5,
3e-4,
1.25,
2e-6,
10,
norm_weight,
7e-6,
)
actual = mhc_shifted_post_pre_deep_gemm(*args)
expected = (
expected_residual,
expected_post_mix,
expected_res_mix,
expected_y_bf16,
expected_previous_mix,
)
for i, (result, ref) in enumerate(zip(actual, expected, strict=True)):
# Match the existing mHC BF16 tolerance: post rounding differences
# can be amplified by cancellation in the carried collapse.
atol, rtol = (1.6e-2, 1e-2) if i in (0, 3) else (1e-6, 1e-3)
torch.testing.assert_close(result, ref, atol=atol, rtol=rtol)
if num_tokens:
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
# DeepGEMM initializes barriers per stream before capture.
mhc_shifted_post_pre_deep_gemm(*args)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph, stream=stream):
captured = mhc_shifted_post_pre_deep_gemm(*args)
for multiplier in (0.5, -1.0):
x.mul_(multiplier)
graph.replay()
eager = mhc_shifted_post_pre_deep_gemm(*args)
for result, ref in zip(captured, eager, strict=True):
torch.testing.assert_close(result, ref, atol=0, rtol=0)
torch.cuda.current_stream().wait_stream(stream)
def sinkhorn_normalize_ref(x: torch.Tensor, repeat: int, eps: float) -> torch.Tensor:
x = x.softmax(-1) + eps
x = x / (x.sum(-2, keepdim=True) + eps)