[Feat] DeepSeek V4 Rebased (#40860)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: qizixi <zixi@inferact.ai>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Simon Mo <simon@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: youkaichao <youkaichao@gmail.com>
Co-authored-by: Zhewen Li <jerven.vllm@gmail.com>
Co-authored-by: Zijing Liu <liuzijing2014@gmail.com>
Co-authored-by: khluu <khluu000@gmail.com>
Co-authored-by: qizixi <zixi@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-04-26 18:31:08 -07:00
committed by GitHub
co-authored by Yongye Zhu Yongye Zhu Simon Mo Bugen Zhao Giancarlo Delfin Jee Jee Li Nick Hill Roger Wang Roy Wang Woosuk Kwon youkaichao Zhewen Li Zijing Liu khluu qizixi Zhewen Li
parent 32e45636e3
commit 4d51588e23
150 changed files with 16312 additions and 716 deletions
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
profile run. These tests pin the new grid layout.
"""
from __future__ import annotations
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.ops.deepseek_v4_ops import fused_q_kv_rmsnorm
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
)
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
x_f32 = x.to(torch.float32)
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
return y.to(x.dtype)
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
torch.manual_seed(0)
device = "cuda"
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
# spot-check a couple of rows against the torch reference
for row in (0, num_tokens // 2, num_tokens - 1):
torch.testing.assert_close(
qr_out[row],
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
torch.testing.assert_close(
kv_out[row],
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)