[Feat] DeepSeek V4 Rebased (#40860)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: qizixi <zixi@inferact.ai>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Simon Mo <simon@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: youkaichao <youkaichao@gmail.com>
Co-authored-by: Zhewen Li <jerven.vllm@gmail.com>
Co-authored-by: Zijing Liu <liuzijing2014@gmail.com>
Co-authored-by: khluu <khluu000@gmail.com>
Co-authored-by: qizixi <zixi@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-04-26 18:31:08 -07:00
committed by GitHub
co-authored by Yongye Zhu Yongye Zhu Simon Mo Bugen Zhao Giancarlo Delfin Jee Jee Li Nick Hill Roger Wang Roy Wang Woosuk Kwon youkaichao Zhewen Li Zijing Liu khluu qizixi Zhewen Li
parent 32e45636e3
commit 4d51588e23
150 changed files with 16312 additions and 716 deletions
@@ -9,8 +9,8 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
calc_diff,
fp8_mqa_logits,
fp8_paged_mqa_logits,
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
)
@@ -127,8 +127,8 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
logits = fp8_mqa_logits(
q_fp8, kv_fp8, weights, ks, ke, clean_logits=clean_logits
logits = fp8_fp4_mqa_logits(
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
)
ref_logits = _ref_fp8_mqa_logits(
@@ -150,7 +150,7 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_paged_mqa_logits(
def _ref_fp8_fp4_paged_mqa_logits(
q: torch.Tensor,
kv_cache: torch.Tensor,
weights: torch.Tensor,
@@ -205,8 +205,10 @@ def _ref_fp8_paged_mqa_logits(
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("clean_logits", [True, False])
def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
def test_deepgemm_fp8_fp4_paged_mqa_logits():
# NOTE: clean_logits=True is incompatible with the 2D context_lens
# required by csrc/apis/attention.hpp; only the False path is exercised.
clean_logits = False
torch.manual_seed(0)
random.seed(0)
@@ -258,21 +260,29 @@ def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
q_fp8 = q.to(torch.float8_e4m3fn)
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
# deep_gemm paged MQA logits requires 2D context_lens of
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
# see indexer.py:607-608. For each batch/next_n token, the
# effective context length is context_lens[b] - next_n + j + 1.
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
schedule_metadata = get_paged_mqa_logits_metadata(
context_lens, blocksize, get_num_sms()
context_lens_2d, blocksize, get_num_sms()
)
logits = fp8_paged_mqa_logits(
q_fp8,
logits = fp8_fp4_paged_mqa_logits(
(q_fp8, None),
kv_cache_fp8,
weights,
context_lens,
context_lens_2d,
block_tables,
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
)
ref_logits = _ref_fp8_paged_mqa_logits(
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
weights,
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
profile run. These tests pin the new grid layout.
"""
from __future__ import annotations
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.ops.deepseek_v4_ops import fused_q_kv_rmsnorm
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
)
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
x_f32 = x.to(torch.float32)
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
return y.to(x.dtype)
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
torch.manual_seed(0)
device = "cuda"
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
# spot-check a couple of rows against the torch reference
for row in (0, num_tokens // 2, num_tokens - 1):
torch.testing.assert_close(
qr_out[row],
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
torch.testing.assert_close(
kv_out[row],
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
+195 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit-test DeepGEMM FP8 kernels (no DeepEP).
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
@@ -21,6 +21,8 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
@@ -204,3 +206,195 @@ def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_i
f"DeepGEMM path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
# ---------------------------------------------------------------------------
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
# ---------------------------------------------------------------------------
def make_mxfp4_weights(
e: int,
n: int,
k: int,
):
"""
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
w2 shape: (E, K, N//2) uint8 — packed FP4
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
w1_bf16: (E, 2N, K) — original BF16 for reference
w2_bf16: (E, K, N) — original BF16 for reference
"""
from deep_gemm.utils.math import per_token_cast_to_fp4
dtype = torch.bfloat16
gran_k = 32 # MXFP4 block size
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
# Quantize per-expert to FP4
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
w1_s = torch.empty(
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
)
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_token_cast_to_fp4(
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
w2[i], w2_s[i] = per_token_cast_to_fp4(
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
"""BF16 token-loop MoE reference for correctness testing."""
import torch.nn.functional as F
num_tokens, hidden_size = x.shape
intermediate = w1.shape[1] // 2
top_k = topk_ids.shape[1]
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
for t in range(num_tokens):
for kk in range(top_k):
e = topk_ids[t, kk].item()
w = topk_weights[t, kk].item()
fc1 = x[t : t + 1].float() @ w1[e].float().T
linear = fc1[:, :intermediate]
gate = fc1[:, intermediate:]
act = F.silu(gate) * linear
fc2 = act @ w2[e].float().T
output[t] += w * fc2[0]
return output.to(torch.bfloat16)
def run_single_fp4_case(m, n, k, topk, num_experts):
"""
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
# FP4 expert weight tensors + BF16 originals for reference
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
)
from vllm.platforms import current_platform
_fp8_dtype = current_platform.fp8_dtype()
_block_shape = GroupShape(128, 128)
quant_config = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
)
moe_config = make_dummy_moe_config()
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmFP4Experts,
)
deep_gemm_fp4_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=DeepGemmFP4Experts(
moe_config=moe_config,
quant_config=quant_config,
),
inplace=False,
)
# DeepGEMM FP4 path
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# BF16 reference using the same original weights
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
diff = calc_diff(out_deepgemm_fp4, out_ref)
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
# FP4 quantization with block_k=32 needs large K for good accuracy.
FP4_MNKs = [
(128, 4096, 4096), # DeepSeek V4 shape
(256, 2048, 2048), # Half-size variant
]
FP4_TOPKS = [2]
FP4_NUM_EXPERTS = [8]
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
@pytest.mark.parametrize("topk", FP4_TOPKS)
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_fp4_vs_triton(
m, n, k, topk, num_experts, monkeypatch, workspace_init
):
pytest.importorskip("deep_gemm.utils.math")
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmFP4Experts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmFP4Experts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmFP4Experts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_fp4_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
)
# ensure that the DeepGEMM FP4 path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM FP4 path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.platforms import current_platform
def _torch_topk_softplus_sqrt(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
if hash_indices_table is not None:
assert input_ids is not None
topk_ids = hash_indices_table[input_ids.long()]
else:
topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=True)[1]
topk_weights = original_scores.gather(1, topk_ids.long())
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def test_sqrtsoftplus_bias_uses_deepseek_v4_routing_method():
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=True,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.DeepseekV4
)
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=False,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.Unspecified
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [128, 256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt_hash(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
vocab_size = 1024
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
# Per-token fixed expert selection: for each vocab id pick `topk` distinct
# experts.
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=torch.int32)
input_ids = torch.randint(
0, vocab_size, (num_tokens,), dtype=torch.int32, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=routed_scaling_factor,
)
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
+311
View File
@@ -0,0 +1,311 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Round-trip tests for compressor → FP8 quant + KV cache insert → gather + dequant.
Two paths tested:
A) DeepseekV4 Attention: head_dim=512 (448 FP8 nope + 64 bf16 rope), quant_block=64
B) Indexer: head_dim=128 (all FP8), quant_block=128
These serve as golden references for validating the future fused
compressor+quant+cache kernel.
"""
import math
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.v1.attention.ops.deepseek_v4_ops import (
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
"""PyTorch reference for UE8M0 FP8 quantization (per-block, power-of-2 scale).
Returns (x_fp8, scales) where x_fp8 is float8_e4m3fn and scales are float32.
"""
assert x.dim() == 1
n = x.numel()
n_blocks = math.ceil(n / block_size)
x_fp8 = torch.zeros(n, dtype=torch.float8_e4m3fn, device=x.device)
scales = torch.zeros(n_blocks, dtype=torch.float32, device=x.device)
for i in range(n_blocks):
start = i * block_size
end = min(start + block_size, n)
block = x[start:end].float()
amax = block.abs().max().clamp(min=1e-4)
raw_scale = amax / fp8_max
exponent = math.ceil(math.log2(raw_scale.item()))
scale = 2.0**exponent
scales[i] = scale
quantized = (block / scale).clamp(-fp8_max, fp8_max)
x_fp8[start:end] = quantized.to(torch.float8_e4m3fn)
return x_fp8, scales
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache
→ compare against original."""
HEAD_DIM = 512
NOPE_DIM = 448
HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale
FP8_MAX = 448.0
QUANT_BLOCK = 64
num_blocks = (num_tokens + block_size - 1) // block_size + 1
device = "cuda"
# Random compressed_kv (simulates compressor output)
compressed_kv = torch.randn(
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
# ── Quant + insert ──────────────────────────────────────────────────
k_cache = torch.zeros(
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
)
k_cache_2d = k_cache.view(num_blocks, -1)
# Sequential slot mapping: token i → slot i
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(
compressed_kv, k_cache_2d, slot_mapping, block_size=block_size
)
# ── Gather + dequant ────────────────────────────────────────────────
num_reqs = 1
max_blocks_per_seq = num_blocks
out = torch.zeros(
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
# block_table: request 0 uses physical blocks 0, 1, ...
block_table = torch.arange(
max_blocks_per_seq, dtype=torch.int32, device=device
).unsqueeze(0)
dequantize_and_gather_k_cache(
out, k_cache, seq_lens, None, block_table, block_size, offset=0
)
recovered = out[0, :num_tokens]
# ── NoPE portion (first 448): FP8 quantized, expect UE8M0 error ──
nope_orig = compressed_kv[:, :NOPE_DIM].float()
nope_recv = recovered[:, :NOPE_DIM].float()
nope_diff = (nope_recv - nope_orig).abs()
# Per-token check: FP8 e4m3 (3-bit mantissa) worst-case error is
# half-ULP at the largest representable value. At y ≈ 448 (max),
# ULP = 2^(8-3) = 32, so error ≤ 16 * scale.
for t in range(num_tokens):
_, scales = _ue8m0_reference(
compressed_kv[t, :NOPE_DIM].float(), QUANT_BLOCK, FP8_MAX
)
max_allowed = 16.0 * scales.max().item()
token_diff = nope_diff[t].max().item()
assert token_diff <= max_allowed, (
f"Token {t} nope diff {token_diff} exceeds max_allowed "
f"{max_allowed} (scale={scales.max().item()})"
)
# ── RoPE portion (last 64): stored as bf16, should be exact ─────
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs()
assert rope_diff.max().item() == 0.0, (
f"RoPE portion should be exact but got max diff {rope_diff.max().item()}"
)
# ── Test B: Indexer path ────────────────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""k → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
→ manual dequant → compare against original."""
HEAD_DIM = 128
QUANT_BLOCK_SIZE = 128
# cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132
CACHE_STRIDE = HEAD_DIM + HEAD_DIM * 4 // QUANT_BLOCK_SIZE
num_blocks = (num_tokens + block_size - 1) // block_size + 1
device = "cuda"
# Random K (simulates compressor output for indexer)
k = torch.randn(num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
# ── Quant + insert ──────────────────────────────────────────────────
kv_cache = torch.zeros(
num_blocks, block_size, CACHE_STRIDE, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
# ── Gather ──────────────────────────────────────────────────────────
max_blocks_per_seq = num_blocks
block_table = torch.arange(
max_blocks_per_seq, dtype=torch.int32, device=device
).unsqueeze(0)
cu_seq_lens = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
# dst_k: [total_seq_len, head_dim] as uint8 (raw FP8 bytes)
dst_k = torch.zeros(num_tokens, HEAD_DIM, dtype=torch.uint8, device=device)
# dst_scale: [total_seq_len, head_dim/quant_block*4] as uint8 (raw float32 bytes)
num_scale_bytes = HEAD_DIM * 4 // QUANT_BLOCK_SIZE # 4
dst_scale = torch.zeros(
num_tokens, num_scale_bytes, dtype=torch.uint8, device=device
)
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
# ── Manual dequant ──────────────────────────────────────────────────
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
k_recovered = k_fp8 * scale # [num_tokens, 128]
# ── Compare ─────────────────────────────────────────────────────────
diff = (k_recovered - k.float()).abs()
k_abs = k.float().abs()
for t in range(num_tokens):
amax = k_abs[t].max().clamp(min=1e-4).item()
# UE8M0: scale = 2^ceil(log2(amax / 448))
exponent = math.ceil(math.log2(amax / 448.0))
ue8m0_scale = 2.0**exponent
# FP8 e4m3 (3-bit mantissa): worst-case error = 16 * scale
max_allowed = 16.0 * ue8m0_scale
token_diff = diff[t].max().item()
assert token_diff <= max_allowed, (
f"Token {t} diff {token_diff} exceeds max_allowed "
f"{max_allowed} (scale={ue8m0_scale})"
)
def test_indexer_gather_accepts_upper_bound_output():
"""Gather only exact cu_seq_lens even when dst is over-allocated."""
head_dim = 128
quant_block_size = 128
cache_stride = head_dim + head_dim * 4 // quant_block_size
valid_tokens = 9
upper_bound_tokens = 13
block_size = 16
num_blocks = 2
sentinel = 123
device = "cuda"
k = torch.randn(valid_tokens, head_dim, dtype=torch.bfloat16, device=device)
kv_cache = torch.zeros(
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
)
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
dst_k = torch.full(
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
)
num_scale_bytes = head_dim * 4 // quant_block_size
dst_scale = torch.full(
(upper_bound_tokens, num_scale_bytes),
sentinel,
dtype=torch.uint8,
device=device,
)
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
torch.accelerator.synchronize()
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
:valid_tokens
].view(torch.float32)
diff = (k_recovered - k.float()).abs()
max_allowed = (16.0 * dst_scale[:valid_tokens].view(torch.float32).max()).item()
assert diff.max().item() <= max_allowed
assert torch.all(dst_k[valid_tokens:] == sentinel)
assert torch.all(dst_scale[valid_tokens:] == sentinel)
# ── Test C: DeepseekV4 attention with values at different magnitudes ───────────
def test_deepseek_v4_quant_magnitude_range():
"""Test that quantization handles a range of magnitudes correctly."""
HEAD_DIM = 512
NOPE_DIM = 448
HEAD_BYTES = 584
block_size = 16
num_tokens = 4
num_blocks = 2
device = "cuda"
# Create inputs with varying magnitudes: small, medium, large
compressed_kv = torch.zeros(
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
compressed_kv[0] = 0.001 # very small
compressed_kv[1] = 1.0 # unit scale
compressed_kv[2] = 100.0 # large
compressed_kv[3] = torch.randn(HEAD_DIM, dtype=torch.bfloat16, device=device)
k_cache = torch.zeros(
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(
compressed_kv, k_cache.view(num_blocks, -1), slot_mapping, block_size
)
out = torch.zeros(1, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
)
dequantize_and_gather_k_cache(
out, k_cache, seq_lens, None, block_table, block_size, offset=0
)
recovered = out[0, :num_tokens]
# RoPE portion must be exact
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs().max()
assert rope_diff.item() == 0.0, f"RoPE diff {rope_diff.item()}"
# NoPE: relative error should be reasonable
for t in range(num_tokens):
orig = compressed_kv[t, :NOPE_DIM].float()
recv = recovered[t, :NOPE_DIM].float()
abs_diff = (recv - orig).abs().max().item()
magnitude = orig.abs().max().item()
if magnitude > 0.01:
rel_err = abs_diff / magnitude
assert rel_err < 0.15, (
f"Token {t}: rel_err={rel_err:.4f}, abs_diff={abs_diff:.6f}, "
f"magnitude={magnitude:.4f}"
)
@@ -0,0 +1,359 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel:
fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert
- Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last 64 dims
- KV side: GPT-J RoPE on last 64 + UE8M0 FP8 quant + paged cache insert
We compare against:
- PyTorch reference for RMSNorm + GPT-J RoPE on Q
- Existing Triton `quantize_and_insert_k_cache` + round-trip via
`dequantize_and_gather_k_cache` for KV
The kernel is imported via
`torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert`.
"""
import pytest
import torch
from vllm.v1.attention.ops.deepseek_v4_ops import (
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
# ── Constants matching the kernel ────────────────────────────────────────────
HEAD_DIM = 512
ROPE_DIM = 64
NOPE_DIM = HEAD_DIM - ROPE_DIM # 448
QUANT_BLOCK = 64
FP8_MAX = 448.0
HEAD_BYTES = NOPE_DIM + ROPE_DIM * 2 + 8 # 448 + 128 + 8 = 584
# ── PyTorch reference implementations ────────────────────────────────────────
def make_cos_sin_cache(max_pos: int, rope_dim: int, dtype, device):
"""Build a cos||sin cache matching DeepseekV4ScalingRotaryEmbedding layout.
cos_sin_cache[pos, :rope_dim/2] = cos(theta), [rope_dim/2:] = sin(theta).
"""
base = 10000.0
inv_freq = 1.0 / (
base
** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)
)
t = torch.arange(max_pos, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j -> ij", t, inv_freq) # [max_pos, rope_dim/2]
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1) # [max_pos, rope_dim]
return cache.to(dtype)
def apply_rope_gptj_last_k(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
"""GPT-J-style (interleaved-pair) RoPE on the LAST rope_dim elements.
x: [..., head_dim] float32
positions: [num_tokens] int64 (positions[i] corresponds to x[i, ...])
cos_sin_cache: [max_pos, rope_dim] float (cos|sin layout)
Returns rotated x (same shape/dtype).
"""
rope_dim = cos_sin_cache.shape[-1]
half = rope_dim // 2
head_dim = x.shape[-1]
nope_dim = head_dim - rope_dim
# Gather cos/sin for each token position: [num_tokens, rope_dim]
cs = cos_sin_cache[positions].to(torch.float32) # [N, rope_dim]
cos = cs[..., :half] # [N, half]
sin = cs[..., half:] # [N, half]
# Reshape leading dims so we can broadcast: x shape [..., head_dim].
# Bring token dim to front; assume x is [num_tokens, ..., head_dim].
# We rely on positions being per-token and all other dims sharing the same pos.
rope = x[..., nope_dim:].float() # [..., rope_dim]
# Make rope pairs: reshape last dim to [half, 2]
shape = rope.shape
rope = rope.reshape(*shape[:-1], half, 2)
even = rope[..., 0] # [..., half]
odd = rope[..., 1]
# Broadcast cos/sin over any heads dim in between. cos/sin are [N, half].
# Add singleton dims for intermediate axes.
for _ in range(rope.ndim - 3):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
new_even = even * cos - odd * sin
new_odd = even * sin + odd * cos
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
out = x.clone().float()
out[..., nope_dim:] = rope_rotated
return out.to(x.dtype)
def rmsnorm_no_weight(x: torch.Tensor, eps: float) -> torch.Tensor:
"""RMSNorm with no learnable weight, matching
`RMSNorm(head_dim, has_weight=False)`."""
orig_dtype = x.dtype
xf = x.float()
variance = xf.pow(2).mean(dim=-1, keepdim=True)
return (xf * torch.rsqrt(variance + eps)).to(orig_dtype)
# ── Dispatch to the CUDA op (skip test cleanly if it isn't built in) ─────────
def _op_available() -> bool:
return hasattr(torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or not _op_available(),
reason="CUDA not available or fused DeepseekV4 op not built in",
)
def _call_fused(q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs):
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs
)
# ── Test 1: Q path numerical parity ──────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("n_heads", [8, 64])
def test_q_path_matches_reference(num_tokens: int, n_heads: int):
torch.manual_seed(0)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
# Reference: RMSNorm (no weight) per head, then GPT-J RoPE on last 64.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
# Fused call with dummy KV tensors (KV branch will write slot_mapping=-1 → noop).
num_blocks = 2
bs = 16
kv = torch.zeros(num_tokens, HEAD_DIM, dtype=dtype, device=device)
k_cache = torch.zeros(
num_blocks, bs, HEAD_BYTES, dtype=torch.uint8, device=device
).view(num_blocks, -1)
slot_mapping = torch.full((num_tokens,), -1, dtype=torch.int64, device=device)
q_fused = q.clone()
_call_fused(q_fused, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs)
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
# ── Test 2: KV path round-trip byte/value parity ─────────────────────────────
def _ue8m0_per_block_scales(kv_roped_nope_f32: torch.Tensor, qblock: int):
"""Return per-token per-block max scale (used to bound FP8 error)."""
n_tok, nope = kv_roped_nope_f32.shape
n_blocks = nope // qblock
blocks = kv_roped_nope_f32.view(n_tok, n_blocks, qblock)
absmax = blocks.abs().amax(dim=-1).clamp(min=1e-4)
raw = absmax / FP8_MAX
exponent = torch.ceil(torch.log2(raw))
return torch.pow(2.0, exponent) # [n_tok, n_blocks]
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_matches_reference(num_tokens: int, block_size: int):
torch.manual_seed(1)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# ── Reference path: RoPE on kv, then existing Triton quant+insert ──────
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# ── Fused path (dummy q, single head) ──────────────────────────────────
k_cache_fused = torch.zeros_like(k_cache_ref)
q_dummy = torch.zeros(num_tokens, 1, HEAD_DIM, dtype=dtype, device=device)
_call_fused(
q_dummy,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
# ── Round-trip compare via dequant+gather ──────────────────────────────
def _dequant(k_cache_2d):
num_reqs = 1
max_blocks = num_blocks
out = torch.zeros(
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.arange(
max_blocks, dtype=torch.int32, device=device
).unsqueeze(0)
# gather_lens arg is None (use seq_lens)
k_cache_3d = k_cache_2d.view(num_blocks, block_size, HEAD_BYTES)
dequantize_and_gather_k_cache(
out, k_cache_3d, seq_lens, None, block_table, block_size, offset=0
)
return out[0, :num_tokens]
recovered_ref = _dequant(k_cache_ref)
recovered_fused = _dequant(k_cache_fused)
# NoPE: per-block UE8M0 FP8 error bound (half-ULP at max = 16 * scale).
scales = _ue8m0_per_block_scales(kv_ref[:, :NOPE_DIM].float(), QUANT_BLOCK)
for t in range(num_tokens):
max_allowed = 16.0 * scales[t].max().item()
diff_ref = (
(recovered_ref[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
)
diff_fused = (
(recovered_fused[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
)
assert diff_ref <= max_allowed, (
f"ref NoPE token {t} diff {diff_ref} > {max_allowed}"
)
assert diff_fused <= max_allowed, (
f"fused NoPE token {t} diff {diff_fused} > {max_allowed}"
)
# RoPE region: bf16 stored exactly → zero diff.
rope_diff = (recovered_fused[:, NOPE_DIM:] - kv_ref[:, NOPE_DIM:]).abs().max()
assert rope_diff.item() == 0.0, f"RoPE portion not exact: {rope_diff.item()}"
# Exact byte equality of the two cache buffers — strong parity.
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
@pytest.mark.parametrize("num_tokens", [4, 17])
@pytest.mark.parametrize("pad", [1, 5])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
tokens in the KV branch while still running Q-norm+RoPE on all rows."""
torch.manual_seed(3)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
total = num_tokens + pad
kv = torch.randn(total, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(total, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# Reference: only the first num_tokens kv rows get inserted.
kv_ref = apply_rope_gptj_last_k(
kv[:num_tokens], positions[:num_tokens], cos_sin_cache
)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# Fused: pass full-sized q/kv/positions, shorter slot_mapping.
q_dummy = torch.zeros(total, 1, HEAD_DIM, dtype=dtype, device=device)
k_cache_fused = torch.zeros_like(k_cache_ref)
_call_fused(
q_dummy,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
# ── Test 3: combined single-call Q + KV parity ───────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17])
@pytest.mark.parametrize("n_heads", [8, 64])
@pytest.mark.parametrize("block_size", [16, 64])
def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
torch.manual_seed(2)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# Reference.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# Fused single call.
q_fused = q.clone()
k_cache_fused = torch.zeros_like(k_cache_ref)
_call_fused(
q_fused,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for fused_indexer_q_rope_quant.
Compares the fused Triton kernel against the unfused reference flow used by
the DeepseekV4 indexer in model_tracking:
q_rot = ops.rotary_embedding(positions, q, None, head_dim, cos_sin_cache,
is_neox_style=False,
rope_dim_offset=head_dim - rope_dim)
q_fp8, q_scale = per_token_group_quant_fp8(q_rot, head_dim, use_ue8m0=True)
weights_out = weights * q_scale * softmax_scale * head_scale
Expects bit-exact equality on both q_fp8 and weights_out.
"""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.v1.attention.ops.deepseek_v4_ops.fused_indexer_q import (
fused_indexer_q_rope_quant,
)
HEAD_DIM = 128
ROPE_DIM = 64
N_HEAD = 64
MAX_POS = 4096
def _reference(
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
softmax_scale: float,
head_scale: float,
) -> tuple[torch.Tensor, torch.Tensor]:
q_rot = q.clone()
ops.rotary_embedding(
positions,
q_rot,
None,
HEAD_DIM,
cos_sin_cache,
False, # is_neox_style=False → GPT-J interleaved
HEAD_DIM - ROPE_DIM, # rope_dim_offset → rotate the tail
False,
)
q_fp8, q_scale = per_token_group_quant_fp8(
q_rot.view(-1, HEAD_DIM).contiguous(),
HEAD_DIM,
use_ue8m0=True,
)
q_fp8 = q_fp8.view(-1, N_HEAD, HEAD_DIM)
q_scale = q_scale.view(-1, N_HEAD)
weights_out = weights.to(torch.float32) * q_scale * softmax_scale * head_scale
return q_fp8, weights_out
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 257])
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
@torch.inference_mode()
def test_fused_indexer_q_rope_quant_matches_unfused(num_tokens, cache_dtype):
device = "cuda"
torch.manual_seed(0)
q = torch.randn(num_tokens, N_HEAD, HEAD_DIM, dtype=torch.bfloat16, device=device)
positions = torch.randint(
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=cache_dtype, device=device)
weights = torch.randn(num_tokens, N_HEAD, dtype=torch.bfloat16, device=device)
softmax_scale = HEAD_DIM**-0.5
head_scale = N_HEAD**-0.5
q_fp8_ref, weights_ref = _reference(
positions, q, cos_sin_cache, weights, softmax_scale, head_scale
)
q_fp8_fused, weights_fused = fused_indexer_q_rope_quant(
positions, q.clone(), cos_sin_cache, weights, softmax_scale, head_scale
)
# fp8 tensors aren't directly comparable via torch.equal — reinterpret as int8.
ref_bits = q_fp8_ref.view(torch.int8)
fused_bits = q_fp8_fused.view(torch.int8)
assert torch.equal(ref_bits, fused_bits), (
f"q_fp8 mismatch: "
f"{(ref_bits != fused_bits).sum().item()} / {ref_bits.numel()} bytes differ"
)
assert torch.equal(weights_ref, weights_fused), (
f"weights mismatch: max abs diff "
f"{(weights_ref - weights_fused).abs().max().item()}"
)
@@ -0,0 +1,908 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the fused inverse RoPE + block-scaled FP8 quantization kernel.
Tests compare the fused kernel against a reference implementation built from
the existing separate operations (inverse RoPE via rotate_neox + FP8 quant
via per_token_group_quant_fp8).
The reference faithfully reproduces the exact flow in deepseek_v4_attention.py:295-310:
1. Apply inverse RoPE (NeoX style, last rope_dim=64 dims of each head)
2. Reshape [T, H, head_dim] -> [T, G, D]
3. Transpose+flatten to [G*T, D], quantize, reshape back
4. Return o_fp8 and o_scale with strides (D, T*D, 1) and (S, T*S, 1)
(non-contiguous [T, G, ...] view backed by contiguous [G, T, ...] memory)
Usage:
pytest tests/kernels/test_fused_inv_rope_fp8_quant.py -v
"""
import pytest
import torch
from vllm.v1.attention.ops.deepseek_v4_ops import fused_inv_rope_fp8_quant
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
NOPE_DIM = 448
ROPE_DIM = 64
QUANT_GROUP_SIZE = 128
FP8_MAX = 448.0 # torch.finfo(torch.float8_e4m3fn).max
FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
# =========================================================================
# Helpers
# =========================================================================
def assert_dequant_close(
fp8_a: torch.Tensor,
scale_a: torch.Tensor,
fp8_b: torch.Tensor,
scale_b: torch.Tensor,
msg: str = "",
):
"""Compare two FP8-quantized tensors via their dequantized values.
Uses cosine-similarity-based diff (same as deep_gemm calc_diff).
Both fused and reference paths rotate in fp32 using an fp32
cos_sin_cache, so differences are only fp32 ordering ULPs that can
occasionally shift FP8 values at quantization boundaries.
"""
S = scale_a.shape[-1]
shape = fp8_a.shape
dq_a = fp8_a.float() * scale_a.unsqueeze(-1).expand(
*shape[:-1], S, QUANT_GROUP_SIZE
).reshape(shape)
dq_b = fp8_b.float() * scale_b.unsqueeze(-1).expand(
*shape[:-1], S, QUANT_GROUP_SIZE
).reshape(shape)
# Cosine diff: 1 - cos_sim (0 = identical, higher = worse)
dq_a_flat = dq_a.flatten().float()
dq_b_flat = dq_b.flatten().float()
cos_sim = torch.nn.functional.cosine_similarity(
dq_a_flat.unsqueeze(0), dq_b_flat.unsqueeze(0)
).item()
diff = 1.0 - cos_sim
assert diff < 1e-4, f"Dequant diff too large: {diff:.8f} (expected < 1e-4). {msg}"
def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
"""GPT-J style rotation: interleaved pairs, negate-swap.
Matches vllm/model_executor/layers/rotary_embedding/common.py:23-27.
DeepseekV4 uses is_neox_style=False, so this is the correct rotation.
"""
x1 = x[..., ::2]
x2 = x[..., 1::2]
x = torch.stack((-x2, x1), dim=-1)
return x.flatten(-2)
def make_cos_sin_cache(
max_pos: int,
rope_dim: int = ROPE_DIM,
dtype: torch.dtype = torch.float32,
device: str = "cuda",
) -> torch.Tensor:
"""Create a synthetic cos_sin_cache matching the layout used by
DeepseekV4ScalingRotaryEmbedding._compute_cos_sin_cache.
Shape: [max_pos, rope_dim] where first half is cos, second half is sin.
The fused kernel requires fp32; callers can override dtype if passing
the cache into the bf16-only paths.
"""
half = rope_dim // 2
# Use random but bounded frequencies so cos/sin are well-behaved
inv_freq = 1.0 / (
10000.0 ** (torch.arange(0, half, device=device, dtype=torch.float32) / half)
)
t = torch.arange(max_pos, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv_freq) # [max_pos, half]
cos = freqs.cos()
sin = freqs.sin()
cache = torch.cat((cos, sin), dim=-1) # [max_pos, rope_dim]
return cache.to(dtype)
def reference_inv_rope(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
) -> torch.Tensor:
"""Apply inverse RoPE to the last rope_dim dimensions of each head.
Matches the GPT-J inverse rotation in pos_encoding_kernels.cu, which
promotes the cache to fp32 and performs the rotation in fp32. The
result is cast back to the input dtype.
Args:
o: [T, H, head_dim] bf16
positions: [T] int64
cos_sin_cache: [max_pos, rope_dim] fp32
Returns:
o with inverse RoPE applied on the rope portion (bf16).
"""
assert cos_sin_cache.dtype == torch.float32
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
half = rope_dim // 2
cos = cos_sin[:, :half]
sin = cos_sin[:, half:]
# GPT-J style: repeat_interleave (not repeat) to match interleaved pairs
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(1)
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(1)
sin = -sin # inverse
o_pass = o[..., :nope_dim]
o_rot_f32 = o[..., nope_dim:].float()
o_rot_f32 = o_rot_f32 * cos + rotate_gptj(o_rot_f32) * sin
o_rot = o_rot_f32.to(o.dtype)
return torch.cat([o_pass, o_rot], dim=-1)
def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-block UE8M0 FP8 quantization in pure float32.
Matches the Triton kernel logic exactly:
absmax -> 2^ceil(log2(absmax / fp8_max)) -> clamp(x / scale) -> fp8
Args:
x_f32: [..., quant_group_size] float32 — one or more 128-element blocks.
Returns:
x_fp8: same shape, float8_e4m3fn
scales: [...] float32, one scale per block
"""
assert x_f32.shape[-1] == QUANT_GROUP_SIZE
absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS)
scale_raw = absmax * (1.0 / FP8_MAX)
scale = torch.exp2(torch.ceil(torch.log2(scale_raw)))
x_scaled = (x_f32 / scale).clamp(-FP8_MAX, FP8_MAX)
x_fp8 = x_scaled.to(FP8_DTYPE)
return x_fp8, scale.squeeze(-1)
def reference_inv_rope_fp8_quant(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
n_groups: int,
heads_per_group: int,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
quant_group_size: int = QUANT_GROUP_SIZE,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full reference: inverse RoPE in fp32 + UE8M0 FP8 quant in fp32.
Mimics the Triton kernel's precision path exactly:
Load bf16 -> cast to fp32 -> apply inverse RoPE with fp32 cos/sin ->
UE8M0 quant in fp32 -> write fp8 + scale
Returns:
o_fp8: [T, G, D] FP8 with strides (D, T*D, 1)
o_scale: [T, G, S] FP32 with strides (S, T*S, 1)
"""
assert cos_sin_cache.dtype == torch.float32
T, _H, head_dim = o.shape
d = heads_per_group * head_dim
S = d // quant_group_size
half_rope = rope_dim // 2
chunks_per_head = head_dim // quant_group_size
# Reshape [T, H, head_dim] -> [T, G, heads_per_group, head_dim]
o_4d = o.view(T, n_groups, heads_per_group, head_dim)
# Lookup cos/sin directly in fp32
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
cos = cos_sin[:, :half_rope] # [T, half_rope] fp32
sin = cos_sin[:, half_rope:] # [T, half_rope] fp32
# Allocate outputs in [G, T, ...] contiguous layout
fp8_buf = torch.empty(n_groups, T, d, dtype=FP8_DTYPE, device=o.device)
scale_buf = torch.empty(n_groups, T, S, dtype=torch.float32, device=o.device)
# Process each quant block, matching the Triton kernel's per-program logic
for g in range(n_groups):
for qb in range(S):
head_in_group = qb // chunks_per_head
chunk_in_head = qb % chunks_per_head
offset = chunk_in_head * quant_group_size
# Load 128 bf16 elements and promote to fp32 for rotation+quant
block = o_4d[:, g, head_in_group, offset : offset + quant_group_size]
x = block.float()
# Apply inverse RoPE in fp32 if this is the last chunk
# GPT-J style: interleaved pairs (even=x, odd=y)
if chunk_in_head == chunks_per_head - 1:
rope_start = nope_dim % quant_group_size # 64
rope_region = x[:, rope_start:].clone()
x_vals = rope_region[:, ::2]
y_vals = rope_region[:, 1::2]
x_new = x_vals * cos + y_vals * sin
y_new = y_vals * cos - x_vals * sin
x = x.clone()
x[:, rope_start::2] = x_new
x[:, rope_start + 1 :: 2] = y_new
# UE8M0 quant in fp32
x_fp8, scale = _ref_ue8m0_quant_block(x)
# Write to [G, T, D] contiguous memory
fp8_buf[g, :, qb * quant_group_size : (qb + 1) * quant_group_size] = x_fp8
scale_buf[g, :, qb] = scale
# Return transposed views
return fp8_buf.transpose(0, 1), scale_buf.transpose(0, 1)
# =========================================================================
# Tests
# =========================================================================
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8), (32, 4), (128, 8)],
ids=["H64_G8", "H32_G4", "H128_G8"],
)
@pytest.mark.parametrize("seed", [0, 42])
@torch.inference_mode()
def test_correctness(num_tokens, num_heads, n_groups, seed):
"""Compare fused kernel against reference for FP8 values and scales."""
torch.manual_seed(seed)
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
# Create inputs
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(
max_pos, ROPE_DIM, dtype=torch.float32, device=device
)
# Reference
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Fused kernel
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Check shapes
d = heads_per_group * HEAD_DIM
S = d // QUANT_GROUP_SIZE
assert ref_fp8.shape == (num_tokens, n_groups, d)
assert fused_fp8.shape == (num_tokens, n_groups, d)
assert ref_scale.shape == (num_tokens, n_groups, S)
assert fused_scale.shape == (num_tokens, n_groups, S)
# Scales: exact match (both use identical UE8M0 algorithm)
# Scales may differ by one UE8M0 step (factor of 2) if fp32 rotation
# ordering shifts absmax across a power-of-2 boundary. Check ratio is
# close to 1.
scale_ratio = fused_scale / ref_scale.clamp(min=1e-30)
assert scale_ratio.max() <= 2.0 and scale_ratio.min() >= 0.5, (
f"Scale ratio out of [0.5, 2]: min={scale_ratio.min():.4f} "
f"max={scale_ratio.max():.4f}"
)
# Compare via dequant (Triton vs PyTorch fp32 may differ by ULPs)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8), (128, 8)],
ids=["H64_G8", "H128_G8"],
)
@torch.inference_mode()
def test_output_strides(num_tokens, num_heads, n_groups):
"""Verify fused output layout:
- FP8: logical [T, G, D] backed by contiguous [G, T, D].
- Scale: MN-major TMA-aligned (column-major: T-stride=1).
"""
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# FP8: logical [T, G, D] backed by [G, T, D] row-major
d = heads_per_group * HEAD_DIM
expected_fp8_stride = (d, num_tokens * d, 1)
assert fused_fp8.stride() == expected_fp8_stride, (
f"FP8 stride mismatch: got {fused_fp8.stride()}, expected {expected_fp8_stride}"
)
# Scale: MN-major TMA-aligned layout. After fp8_einsum permutes
# [T,G,S] -> [G,T,S], T-dim should have stride 1.
# Our output is [T,G,S] = transpose of [G,T,S].
# So fused_scale.permute(1,0,2) should have T-stride=1.
perm = fused_scale.permute(1, 0, 2) # [G, T, S]
assert perm.stride(1) == 1 or num_tokens == 1, (
f"Scale T-stride (after permute to [G,T,S]) should be 1, got {perm.stride(1)}"
)
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@torch.inference_mode()
def test_per_group_contiguity(num_tokens):
"""FP8 per-group slices must be contiguous. Scale per-group slices
are column-major (T-stride=1) — not row-major contiguous, which is
correct for TMA loads."""
num_heads, n_groups = 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
for g in range(n_groups):
fp8_slice = fused_fp8[:, g, :]
assert fp8_slice.is_contiguous(), (
f"o_fp8[:, {g}, :] is not contiguous: "
f"shape={list(fp8_slice.shape)}, stride={list(fp8_slice.stride())}"
)
@torch.inference_mode()
def test_scales_are_power_of_two():
"""Verify all scales are exact powers of 2 (UE8M0 property)."""
num_tokens, num_heads, n_groups = 32, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
_, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# log2 of a power-of-two is an exact integer
log2_scales = torch.log2(fused_scale)
residual = (log2_scales - log2_scales.round()).abs()
assert residual.max() < 1e-5, (
f"Not all scales are powers of 2: max log2 residual = {residual.max().item()}"
)
@torch.inference_mode()
def test_nope_dims_unchanged():
"""Nope dimensions (first 448 per head) should only be quantized,
not rotated. Verify by dequantizing and comparing against
quantize-only reference (no RoPE)."""
num_tokens, num_heads, n_groups = 16, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# Fused kernel result
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Reference: quantize without RoPE (identity rotation)
# Create a zero-sin cache so RoPE is identity
zero_cache = torch.zeros_like(cos_sin_cache)
half = ROPE_DIM // 2
zero_cache[:, :half] = 1.0 # cos = 1
# sin = 0 (already zero)
norope_fp8, norope_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
zero_cache,
n_groups,
heads_per_group,
)
# Extract nope quant blocks only (first 3 of every 4 blocks per head)
chunks_per_head = HEAD_DIM // QUANT_GROUP_SIZE # 4
for h in range(heads_per_group):
for c in range(chunks_per_head - 1): # skip last chunk (has rope)
qb = h * chunks_per_head + c
start = qb * QUANT_GROUP_SIZE
end = start + QUANT_GROUP_SIZE
fused_nope = fused_fp8[:, :, start:end].view(torch.uint8)
norope_nope = norope_fp8[:, :, start:end].view(torch.uint8)
assert torch.equal(fused_nope, norope_nope), (
f"Nope block (head={h}, chunk={c}) differs between "
f"fused and no-rope reference"
)
fused_s = fused_scale[:, :, qb]
norope_s = norope_scale[:, :, qb]
assert torch.equal(fused_s, norope_s), (
f"Nope scale (head={h}, chunk={c}) differs"
)
@torch.inference_mode()
def test_single_token():
"""Edge case: single token."""
num_tokens, num_heads, n_groups = 1, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.tensor([42], device=device, dtype=torch.long)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_zero_positions():
"""Edge case: all positions are 0."""
num_tokens, num_heads, n_groups = 16, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.zeros(num_tokens, device=device, dtype=torch.long)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_large_values():
"""Edge case: values near FP8 saturation to test clamping."""
num_tokens, num_heads, n_groups = 8, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
# Create inputs with large values that will saturate FP8
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
o = o * 1000.0 # scale up to force saturation
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_dequant_numerical_accuracy():
"""Verify dequantized values are close to the original (after inv RoPE)."""
num_tokens, num_heads, n_groups = 32, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# Get the post-inv-RoPE values (ground truth before quantization)
o_after_rope = reference_inv_rope(o.clone(), positions, cos_sin_cache)
d = heads_per_group * HEAD_DIM
o_after_rope = o_after_rope.view(num_tokens, n_groups, d)
# Get fused quantized output
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Dequantize: broadcast scale [T, G, S] to [T, G, D] via repeat
S = d // QUANT_GROUP_SIZE
scale_expanded = (
fused_scale.unsqueeze(-1)
.expand(num_tokens, n_groups, S, QUANT_GROUP_SIZE)
.reshape(num_tokens, n_groups, d)
)
dequant = fused_fp8.float() * scale_expanded
# Check relative error.
# FP8 e4m3 with UE8M0 (power-of-two scales that round UP) quantizes more
# coarsely than optimal scaling. Both paths rotate in fp32, so the bulk
# of the error comes from UE8M0 quantization itself (~10-12% typical).
o_gt = o_after_rope.transpose(0, 1).contiguous().transpose(0, 1)
dequant_contig = dequant.transpose(0, 1).contiguous().transpose(0, 1)
abs_err = (dequant_contig.float() - o_gt.float()).abs()
rel_err = abs_err / (o_gt.float().abs().clamp(min=1e-6))
mean_rel_err = rel_err.mean().item()
assert mean_rel_err < 0.15, (
f"Mean relative error too high: {mean_rel_err:.4f} (expected < 0.15)"
)
def _unfused_inv_rope_fp8_quant(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
n_groups: int,
heads_per_group: int,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Unfused path matching deepseek_v4_attention.py:295-310.
Uses the production CUDA RoPE kernel + per_token_group_quant_fp8.
"""
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
head_dim = o.shape[-1]
rope_dim_offset = head_dim - rope_dim
# Step 1: In-place CUDA RoPE (same as production)
ops.rotary_embedding(
positions,
o,
None,
head_dim,
cos_sin_cache,
False, # is_neox=False for DeepseekV4 (GPT-J style)
rope_dim_offset=rope_dim_offset,
inverse=True,
)
# Step 2: Reshape + quant + reshape (same as production)
T = o.shape[0]
d = heads_per_group * head_dim
o = o.view(T, n_groups, -1)
o_flat = o.transpose(0, 1).contiguous().reshape(-1, d)
o_fp8, o_scale = per_token_group_quant_fp8(
o_flat,
group_size=QUANT_GROUP_SIZE,
use_ue8m0=True,
)
o_fp8 = o_fp8.view(n_groups, T, d).transpose(0, 1)
o_scale = o_scale.view(n_groups, T, -1).transpose(0, 1)
return o_fp8, o_scale
# =========================================================================
# End-to-end test including fp8_einsum
# =========================================================================
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128, 1024])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8)],
ids=["H64_G8"],
)
@torch.inference_mode()
def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
"""End-to-end: fused inv_rope+quant → fp8_einsum must match
unfused CUDA_rope+quant → fp8_einsum bitwise.
This catches stride/layout bugs that only manifest when the einsum
kernel actually consumes the quantized activations.
"""
from deep_gemm.utils.math import ceil_div
from vllm.utils.deep_gemm import (
fp8_einsum,
per_block_cast_to_fp8,
transform_sf_into_required_layout,
)
heads_per_group = num_heads // n_groups
d = heads_per_group * HEAD_DIM
o_lora_rank = 1024
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# -- Weight quantization (shared between both paths) --
w = torch.randn(n_groups, o_lora_rank, d, device=device, dtype=torch.bfloat16)
w_fp8 = torch.empty_like(w, dtype=torch.float8_e4m3fn)
w_scale = torch.empty(
n_groups,
ceil_div(o_lora_rank, 128),
ceil_div(d, 128),
device=device,
dtype=torch.float32,
)
for g in range(n_groups):
w_fp8[g], w_scale[g] = per_block_cast_to_fp8(w[g], use_ue8m0=True)
recipe = (1, 1, 128)
w_scale_t = transform_sf_into_required_layout(
sf=w_scale,
mn=o_lora_rank,
k=d,
recipe=(1, 128, 128),
num_groups=n_groups,
is_sfa=False,
)
# -- UNFUSED path --
ref_fp8, ref_scale = _unfused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
z_ref = torch.empty(
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
)
fp8_einsum(
"bhr,hdr->bhd", (ref_fp8, ref_scale), (w_fp8, w_scale_t), z_ref, recipe=recipe
)
# -- FUSED path --
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
z_fused = torch.empty(
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
)
fp8_einsum(
"bhr,hdr->bhd",
(fused_fp8, fused_scale),
(w_fp8, w_scale_t),
z_fused,
recipe=recipe,
)
# -- Checks --
# Einsum output: Triton and CUDA both rotate in fp32 now, so diffs
# come from fp32 ordering and UE8M0 boundary shifts only.
# Use relative diff (same metric as test_fp8_einsum.py).
from deep_gemm.testing import calc_diff
z_diff = calc_diff(z_fused, z_ref)
assert z_diff < 0.01, (
f"Einsum output diff too large: {z_diff:.6f} (expected < 0.01)"
)
@pytest.mark.parametrize("num_tokens", [1, 32, 256])
@torch.inference_mode()
def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config):
"""Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style,
mscale=0, YaRN scaling) matching the production config."""
num_heads = 64
n_groups = 8
heads_per_group = num_heads // n_groups
device = "cuda"
torch.manual_seed(0)
# Build YaRN-scaled cos_sin_cache matching real DeepSeek V3/V4 config
# (mscale=0 → mscale=1.0, so no magnitude scaling)
from vllm.model_executor.layers.rotary_embedding.common import (
yarn_find_correction_range,
yarn_linear_ramp_mask,
)
scaling_factor = 16
base = 10000.0
max_pos = 65536
beta_fast, beta_slow = 32, 1
pos_freqs = base ** (
torch.arange(0, ROPE_DIM, 2, dtype=torch.float32, device=device) / ROPE_DIM
)
inv_freq_extra = 1.0 / pos_freqs
inv_freq_interp = 1.0 / (scaling_factor * pos_freqs)
low, high = yarn_find_correction_range(
beta_fast, beta_slow, ROPE_DIM, base, max_pos
)
mask = 1 - yarn_linear_ramp_mask(low, high, ROPE_DIM // 2, dtype=torch.float32).to(
device
)
inv_freq = inv_freq_interp * (1 - mask) + inv_freq_extra * mask
t = torch.arange(max_pos * scaling_factor, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv_freq)
# mscale=0 → yarn_get_mscale returns 1.0
cos_sin_cache = torch.cat([freqs.cos(), freqs.sin()], dim=-1) # fp32
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(0, 4096, (num_tokens,), device=device, dtype=torch.long)
# UNFUSED: CUDA RoPE with is_neox=False (GPT-J)
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
o_unfused = o.clone()
ops.rotary_embedding(
positions,
o_unfused,
None,
HEAD_DIM,
cos_sin_cache,
False, # is_neox=False (GPT-J style)
rope_dim_offset=NOPE_DIM,
inverse=True,
)
d = heads_per_group * HEAD_DIM
T = num_tokens
o_unfused = o_unfused.view(T, n_groups, d)
o_flat = o_unfused.transpose(0, 1).contiguous().reshape(-1, d)
ref_fp8, ref_scale = per_token_group_quant_fp8(
o_flat,
group_size=QUANT_GROUP_SIZE,
use_ue8m0=True,
)
ref_fp8 = ref_fp8.view(n_groups, T, d).transpose(0, 1)
ref_scale = ref_scale.view(n_groups, T, -1).transpose(0, 1)
# FUSED: use the real YaRN-scaled cos_sin_cache
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Scales must match exactly (same UE8M0 algorithm)
# Compare via dequant (Triton bf16 rotation may differ from CUDA by 1 ULP)
assert_dequant_close(
ref_fp8, ref_scale, fused_fp8, fused_scale, msg="Real DeepSeek V4 rope"
)
+5 -10
View File
@@ -718,7 +718,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [2000, 6000, 30000, 80000],
"top_k": 2048,
"data_type": "random",
},
id="mixed_all_paths",
@@ -727,7 +726,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [2048, 4096, 8192, 16000],
"top_k": 2048,
"data_type": "random",
},
id="all_decode_medium",
@@ -736,7 +734,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [70000, 100000, 163840],
"top_k": 2048,
"data_type": "random",
},
id="all_large",
@@ -745,7 +742,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [32767, 32768, 32769, 32772],
"top_k": 2048,
"data_type": "random",
},
id="large_threshold_boundary",
@@ -754,7 +750,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [5000],
"top_k": 2048,
"data_type": "random",
},
id="single_row_medium",
@@ -772,15 +767,15 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [100, 2048, 10000, 80000],
"top_k": 2048,
"data_type": "random",
},
id="trivial_medium_large_mix",
),
],
)
@pytest.mark.parametrize("top_k", [512, 2048])
@torch.inference_mode()
def test_persistent_topk(test_config: dict) -> None:
def test_persistent_topk(test_config: dict, top_k: int) -> None:
"""
Tests specific to the persistent_topk kernel:
- Mixed medium/large rows in the same batch (dynamic per-row dispatch)
@@ -790,14 +785,15 @@ def test_persistent_topk(test_config: dict) -> None:
run_large_context_topk_test(
batch_size=len(test_config["seq_lens"]),
seq_lens=test_config["seq_lens"],
top_k=test_config["top_k"],
top_k=top_k,
data_type=test_config.get("data_type", "random"),
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("top_k", [512, 2048])
@torch.inference_mode()
def test_persistent_topk_padded_stride() -> None:
def test_persistent_topk_padded_stride(top_k: int) -> None:
"""
Test persistent_topk with padded logits (large stride, small seq_len)
to simulate the e2e CUDAGraph scenario where fp8_paged_mqa_logits
@@ -806,7 +802,6 @@ def test_persistent_topk_padded_stride() -> None:
set_random_seed(42)
torch.set_default_device("cuda:0")
top_k = 2048
batch_size = 4
padded_stride = 163840 # DeepSeek-V3.2 max_model_len
actual_seq_lens = [3000, 5000, 8000, 12000]