[Feat] DeepSeek V4 Rebased (#40860)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Signed-off-by: qizixi <zixi@inferact.ai> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Yongye Zhu <yongye@inferact.ai> Co-authored-by: Simon Mo <simon@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Zhewen Li <jerven.vllm@gmail.com> Co-authored-by: Zijing Liu <liuzijing2014@gmail.com> Co-authored-by: khluu <khluu000@gmail.com> Co-authored-by: qizixi <zixi@inferact.ai> Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
co-authored by
Yongye Zhu
Yongye Zhu
Simon Mo
Bugen Zhao
Giancarlo Delfin
Jee Jee Li
Nick Hill
Roger Wang
Roy Wang
Woosuk Kwon
youkaichao
Zhewen Li
Zijing Liu
khluu
qizixi
Zhewen Li
parent
32e45636e3
commit
4d51588e23
@@ -9,8 +9,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_ceil_to_ue8m0,
|
||||
calc_diff,
|
||||
fp8_mqa_logits,
|
||||
fp8_paged_mqa_logits,
|
||||
fp8_fp4_mqa_logits,
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
get_num_sms,
|
||||
get_paged_mqa_logits_metadata,
|
||||
)
|
||||
@@ -127,8 +127,8 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
|
||||
logits = fp8_mqa_logits(
|
||||
q_fp8, kv_fp8, weights, ks, ke, clean_logits=clean_logits
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
|
||||
)
|
||||
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
@@ -150,7 +150,7 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
def _ref_fp8_paged_mqa_logits(
|
||||
def _ref_fp8_fp4_paged_mqa_logits(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
@@ -205,8 +205,10 @@ def _ref_fp8_paged_mqa_logits(
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
@pytest.mark.parametrize("clean_logits", [True, False])
|
||||
def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
|
||||
def test_deepgemm_fp8_fp4_paged_mqa_logits():
|
||||
# NOTE: clean_logits=True is incompatible with the 2D context_lens
|
||||
# required by csrc/apis/attention.hpp; only the False path is exercised.
|
||||
clean_logits = False
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
|
||||
@@ -258,21 +260,29 @@ def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
|
||||
|
||||
# deep_gemm paged MQA logits requires 2D context_lens of
|
||||
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
|
||||
# see indexer.py:607-608. For each batch/next_n token, the
|
||||
# effective context length is context_lens[b] - next_n + j + 1.
|
||||
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
|
||||
context_lens_2d = (
|
||||
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
|
||||
).contiguous()
|
||||
schedule_metadata = get_paged_mqa_logits_metadata(
|
||||
context_lens, blocksize, get_num_sms()
|
||||
context_lens_2d, blocksize, get_num_sms()
|
||||
)
|
||||
logits = fp8_paged_mqa_logits(
|
||||
q_fp8,
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(q_fp8, None),
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
context_lens,
|
||||
context_lens_2d,
|
||||
block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
)
|
||||
|
||||
ref_logits = _ref_fp8_paged_mqa_logits(
|
||||
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
|
||||
q,
|
||||
kv_cache,
|
||||
weights,
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
|
||||
|
||||
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
|
||||
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
|
||||
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
|
||||
profile run. These tests pin the new grid layout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import fused_q_kv_rmsnorm
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
|
||||
)
|
||||
|
||||
|
||||
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
x_f32 = x.to(torch.float32)
|
||||
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
|
||||
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
|
||||
return y.to(x.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
q_size, kv_size = 192, 576
|
||||
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
|
||||
qw = torch.randn(q_size, dtype=dtype, device=device)
|
||||
kvw = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
eps = 1e-6
|
||||
|
||||
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
|
||||
|
||||
qr_ref = _ref_rmsnorm(qr, qw, eps)
|
||||
kv_ref = _ref_rmsnorm(kv, kvw, eps)
|
||||
|
||||
tol = dict(rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(qr_out, qr_ref, **tol)
|
||||
torch.testing.assert_close(kv_out, kv_ref, **tol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
|
||||
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
||||
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
||||
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
||||
lifts that bound to 2**31-1."""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
|
||||
qw = torch.randn(q_size, dtype=dtype, device=device)
|
||||
kvw = torch.randn(kv_size, dtype=dtype, device=device)
|
||||
|
||||
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
|
||||
# spot-check a couple of rows against the torch reference
|
||||
for row in (0, num_tokens // 2, num_tokens - 1):
|
||||
torch.testing.assert_close(
|
||||
qr_out[row],
|
||||
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
kv_out[row],
|
||||
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit-test DeepGEMM FP8 kernels (no DeepEP).
|
||||
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
|
||||
"""
|
||||
|
||||
@@ -21,6 +21,8 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
FusedMoEQuantDesc,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
@@ -204,3 +206,195 @@ def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_i
|
||||
f"DeepGEMM path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mxfp4_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""
|
||||
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
plus BF16 reference weights for validation.
|
||||
|
||||
w1 shape: (E, 2N, K//2) uint8 — packed FP4
|
||||
w2 shape: (E, K, N//2) uint8 — packed FP4
|
||||
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
|
||||
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
|
||||
w1_bf16: (E, 2N, K) — original BF16 for reference
|
||||
w2_bf16: (E, K, N) — original BF16 for reference
|
||||
"""
|
||||
from deep_gemm.utils.math import per_token_cast_to_fp4
|
||||
|
||||
dtype = torch.bfloat16
|
||||
gran_k = 32 # MXFP4 block size
|
||||
|
||||
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
|
||||
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
|
||||
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
|
||||
|
||||
# Quantize per-expert to FP4
|
||||
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
|
||||
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
|
||||
w1_s = torch.empty(
|
||||
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
|
||||
)
|
||||
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
|
||||
|
||||
for i in range(e):
|
||||
w1[i], w1_s[i] = per_token_cast_to_fp4(
|
||||
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
w2[i], w2_s[i] = per_token_cast_to_fp4(
|
||||
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
|
||||
)
|
||||
|
||||
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
|
||||
|
||||
|
||||
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
|
||||
"""BF16 token-loop MoE reference for correctness testing."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
num_tokens, hidden_size = x.shape
|
||||
intermediate = w1.shape[1] // 2
|
||||
top_k = topk_ids.shape[1]
|
||||
|
||||
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
|
||||
for t in range(num_tokens):
|
||||
for kk in range(top_k):
|
||||
e = topk_ids[t, kk].item()
|
||||
w = topk_weights[t, kk].item()
|
||||
fc1 = x[t : t + 1].float() @ w1[e].float().T
|
||||
linear = fc1[:, :intermediate]
|
||||
gate = fc1[:, intermediate:]
|
||||
act = F.silu(gate) * linear
|
||||
fc2 = act @ w2[e].float().T
|
||||
output[t] += w * fc2[0]
|
||||
return output.to(torch.bfloat16)
|
||||
|
||||
|
||||
def run_single_fp4_case(m, n, k, topk, num_experts):
|
||||
"""
|
||||
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
DeepGEMM FP4 == BF16 reference within tolerance.
|
||||
"""
|
||||
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
|
||||
|
||||
# FP4 expert weight tensors + BF16 originals for reference
|
||||
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
|
||||
|
||||
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
|
||||
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
|
||||
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_fp8_dtype = current_platform.fp8_dtype()
|
||||
_block_shape = GroupShape(128, 128)
|
||||
quant_config = FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
|
||||
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
|
||||
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
|
||||
)
|
||||
moe_config = make_dummy_moe_config()
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
DeepGemmFP4Experts,
|
||||
)
|
||||
|
||||
deep_gemm_fp4_experts = mk.FusedMoEKernel(
|
||||
prepare_finalize=maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
fused_experts=DeepGemmFP4Experts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
),
|
||||
inplace=False,
|
||||
)
|
||||
|
||||
# DeepGEMM FP4 path
|
||||
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
|
||||
hidden_states=tokens_bf16,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
global_num_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
apply_router_weight_on_input=False,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
# BF16 reference using the same original weights
|
||||
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
|
||||
|
||||
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
|
||||
diff = calc_diff(out_deepgemm_fp4, out_ref)
|
||||
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
|
||||
|
||||
|
||||
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
|
||||
# FP4 quantization with block_k=32 needs large K for good accuracy.
|
||||
FP4_MNKs = [
|
||||
(128, 4096, 4096), # DeepSeek V4 shape
|
||||
(256, 2048, 2048), # Half-size variant
|
||||
]
|
||||
|
||||
FP4_TOPKS = [2]
|
||||
FP4_NUM_EXPERTS = [8]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
|
||||
@pytest.mark.parametrize("topk", FP4_TOPKS)
|
||||
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_fp4_vs_triton(
|
||||
m, n, k, topk, num_experts, monkeypatch, workspace_init
|
||||
):
|
||||
pytest.importorskip("deep_gemm.utils.math")
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
_DeepGemmFP4Experts = importlib.import_module(
|
||||
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
|
||||
).DeepGemmFP4Experts
|
||||
|
||||
call_counter = {"cnt": 0}
|
||||
|
||||
orig_fn = _DeepGemmFP4Experts.apply
|
||||
|
||||
def _spy_apply(*args, **kwargs):
|
||||
call_counter["cnt"] += 1
|
||||
return orig_fn(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
|
||||
if topk > num_experts:
|
||||
pytest.skip(f"topk={topk} > num_experts={num_experts}")
|
||||
|
||||
run_single_fp4_case(
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
topk=topk,
|
||||
num_experts=num_experts,
|
||||
)
|
||||
|
||||
# ensure that the DeepGEMM FP4 path was indeed taken.
|
||||
assert call_counter["cnt"] == 1, (
|
||||
f"DeepGEMM FP4 path was not executed during the test. "
|
||||
f"Call counter: {call_counter['cnt']}"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
fused_topk_bias,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _torch_topk_softplus_sqrt(
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
input_ids: torch.Tensor | None = None,
|
||||
hash_indices_table: torch.Tensor | None = None,
|
||||
):
|
||||
scores = F.softplus(gating_output.float()).sqrt()
|
||||
original_scores = scores
|
||||
if e_score_correction_bias is not None:
|
||||
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
|
||||
else:
|
||||
scores_for_choice = scores
|
||||
|
||||
if hash_indices_table is not None:
|
||||
assert input_ids is not None
|
||||
topk_ids = hash_indices_table[input_ids.long()]
|
||||
else:
|
||||
topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=True)[1]
|
||||
|
||||
topk_weights = original_scores.gather(1, topk_ids.long())
|
||||
if renormalize:
|
||||
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
|
||||
if routed_scaling_factor != 1.0:
|
||||
topk_weights = topk_weights * routed_scaling_factor
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
def test_sqrtsoftplus_bias_uses_deepseek_v4_routing_method():
|
||||
assert (
|
||||
get_routing_method_type(
|
||||
scoring_func="sqrtsoftplus",
|
||||
top_k=8,
|
||||
renormalize=True,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
== RoutingMethodType.DeepseekV4
|
||||
)
|
||||
assert (
|
||||
get_routing_method_type(
|
||||
scoring_func="sqrtsoftplus",
|
||||
top_k=8,
|
||||
renormalize=False,
|
||||
num_expert_group=None,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
== RoutingMethodType.Unspecified
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [128, 256, 384, 512])
|
||||
@pytest.mark.parametrize("topk", [6, 8, 16])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.5])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_softplus_sqrt(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
e_score_correction_bias = torch.randn(
|
||||
(num_experts,), dtype=torch.float32, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
# Different kernels may return the topk experts in different orders when
|
||||
# scores tie; sort by expert id before comparing.
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [1024, 2048])
|
||||
@pytest.mark.parametrize("num_experts", [256, 384, 512])
|
||||
@pytest.mark.parametrize("topk", [6, 8, 16])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
|
||||
def test_fused_topk_softplus_sqrt_hash(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
routed_scaling_factor: float,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
vocab_size = 1024
|
||||
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
|
||||
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
|
||||
# Per-token fixed expert selection: for each vocab id pick `topk` distinct
|
||||
# experts.
|
||||
hash_indices_table = torch.stack(
|
||||
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
|
||||
).to(device="cuda", dtype=torch.int32)
|
||||
input_ids = torch.randint(
|
||||
0, vocab_size, (num_tokens,), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
input_ids=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
)
|
||||
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
hidden_states=hidden_states,
|
||||
gating_output=gating_output,
|
||||
scoring_func="sqrtsoftplus",
|
||||
e_score_correction_bias=None,
|
||||
topk=topk,
|
||||
renormalize=renormalize,
|
||||
input_tokens=input_ids,
|
||||
hash_indices_table=hash_indices_table,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
|
||||
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
|
||||
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
|
||||
|
||||
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
|
||||
sorted_w = topk_weights.gather(1, idx_ops)
|
||||
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
|
||||
@@ -0,0 +1,311 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Round-trip tests for compressor → FP8 quant + KV cache insert → gather + dequant.
|
||||
|
||||
Two paths tested:
|
||||
A) DeepseekV4 Attention: head_dim=512 (448 FP8 nope + 64 bf16 rope), quant_block=64
|
||||
B) Indexer: head_dim=128 (all FP8), quant_block=128
|
||||
|
||||
These serve as golden references for validating the future fused
|
||||
compressor+quant+cache kernel.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import (
|
||||
dequantize_and_gather_k_cache,
|
||||
quantize_and_insert_k_cache,
|
||||
)
|
||||
|
||||
|
||||
def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
|
||||
"""PyTorch reference for UE8M0 FP8 quantization (per-block, power-of-2 scale).
|
||||
|
||||
Returns (x_fp8, scales) where x_fp8 is float8_e4m3fn and scales are float32.
|
||||
"""
|
||||
assert x.dim() == 1
|
||||
n = x.numel()
|
||||
n_blocks = math.ceil(n / block_size)
|
||||
x_fp8 = torch.zeros(n, dtype=torch.float8_e4m3fn, device=x.device)
|
||||
scales = torch.zeros(n_blocks, dtype=torch.float32, device=x.device)
|
||||
|
||||
for i in range(n_blocks):
|
||||
start = i * block_size
|
||||
end = min(start + block_size, n)
|
||||
block = x[start:end].float()
|
||||
amax = block.abs().max().clamp(min=1e-4)
|
||||
raw_scale = amax / fp8_max
|
||||
exponent = math.ceil(math.log2(raw_scale.item()))
|
||||
scale = 2.0**exponent
|
||||
scales[i] = scale
|
||||
quantized = (block / scale).clamp(-fp8_max, fp8_max)
|
||||
x_fp8[start:end] = quantized.to(torch.float8_e4m3fn)
|
||||
|
||||
return x_fp8, scales
|
||||
|
||||
|
||||
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
"""compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache
|
||||
→ compare against original."""
|
||||
|
||||
HEAD_DIM = 512
|
||||
NOPE_DIM = 448
|
||||
HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale
|
||||
FP8_MAX = 448.0
|
||||
QUANT_BLOCK = 64
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
device = "cuda"
|
||||
|
||||
# Random compressed_kv (simulates compressor output)
|
||||
compressed_kv = torch.randn(
|
||||
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
# ── Quant + insert ──────────────────────────────────────────────────
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
k_cache_2d = k_cache.view(num_blocks, -1)
|
||||
|
||||
# Sequential slot mapping: token i → slot i
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
quantize_and_insert_k_cache(
|
||||
compressed_kv, k_cache_2d, slot_mapping, block_size=block_size
|
||||
)
|
||||
|
||||
# ── Gather + dequant ────────────────────────────────────────────────
|
||||
num_reqs = 1
|
||||
max_blocks_per_seq = num_blocks
|
||||
out = torch.zeros(
|
||||
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
# block_table: request 0 uses physical blocks 0, 1, ...
|
||||
block_table = torch.arange(
|
||||
max_blocks_per_seq, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
|
||||
dequantize_and_gather_k_cache(
|
||||
out, k_cache, seq_lens, None, block_table, block_size, offset=0
|
||||
)
|
||||
|
||||
recovered = out[0, :num_tokens]
|
||||
|
||||
# ── NoPE portion (first 448): FP8 quantized, expect UE8M0 error ──
|
||||
nope_orig = compressed_kv[:, :NOPE_DIM].float()
|
||||
nope_recv = recovered[:, :NOPE_DIM].float()
|
||||
nope_diff = (nope_recv - nope_orig).abs()
|
||||
|
||||
# Per-token check: FP8 e4m3 (3-bit mantissa) worst-case error is
|
||||
# half-ULP at the largest representable value. At y ≈ 448 (max),
|
||||
# ULP = 2^(8-3) = 32, so error ≤ 16 * scale.
|
||||
for t in range(num_tokens):
|
||||
_, scales = _ue8m0_reference(
|
||||
compressed_kv[t, :NOPE_DIM].float(), QUANT_BLOCK, FP8_MAX
|
||||
)
|
||||
max_allowed = 16.0 * scales.max().item()
|
||||
token_diff = nope_diff[t].max().item()
|
||||
assert token_diff <= max_allowed, (
|
||||
f"Token {t} nope diff {token_diff} exceeds max_allowed "
|
||||
f"{max_allowed} (scale={scales.max().item()})"
|
||||
)
|
||||
|
||||
# ── RoPE portion (last 64): stored as bf16, should be exact ─────
|
||||
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs()
|
||||
assert rope_diff.max().item() == 0.0, (
|
||||
f"RoPE portion should be exact but got max diff {rope_diff.max().item()}"
|
||||
)
|
||||
|
||||
|
||||
# ── Test B: Indexer path ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
"""k → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
|
||||
→ manual dequant → compare against original."""
|
||||
|
||||
HEAD_DIM = 128
|
||||
QUANT_BLOCK_SIZE = 128
|
||||
# cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132
|
||||
CACHE_STRIDE = HEAD_DIM + HEAD_DIM * 4 // QUANT_BLOCK_SIZE
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
device = "cuda"
|
||||
|
||||
# Random K (simulates compressor output for indexer)
|
||||
k = torch.randn(num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
|
||||
# ── Quant + insert ──────────────────────────────────────────────────
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, block_size, CACHE_STRIDE, dtype=torch.uint8, device=device
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
|
||||
|
||||
# ── Gather ──────────────────────────────────────────────────────────
|
||||
max_blocks_per_seq = num_blocks
|
||||
block_table = torch.arange(
|
||||
max_blocks_per_seq, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
cu_seq_lens = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
|
||||
|
||||
# dst_k: [total_seq_len, head_dim] as uint8 (raw FP8 bytes)
|
||||
dst_k = torch.zeros(num_tokens, HEAD_DIM, dtype=torch.uint8, device=device)
|
||||
# dst_scale: [total_seq_len, head_dim/quant_block*4] as uint8 (raw float32 bytes)
|
||||
num_scale_bytes = HEAD_DIM * 4 // QUANT_BLOCK_SIZE # 4
|
||||
dst_scale = torch.zeros(
|
||||
num_tokens, num_scale_bytes, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
|
||||
)
|
||||
|
||||
# ── Manual dequant ──────────────────────────────────────────────────
|
||||
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
|
||||
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
|
||||
k_recovered = k_fp8 * scale # [num_tokens, 128]
|
||||
|
||||
# ── Compare ─────────────────────────────────────────────────────────
|
||||
diff = (k_recovered - k.float()).abs()
|
||||
k_abs = k.float().abs()
|
||||
|
||||
for t in range(num_tokens):
|
||||
amax = k_abs[t].max().clamp(min=1e-4).item()
|
||||
# UE8M0: scale = 2^ceil(log2(amax / 448))
|
||||
exponent = math.ceil(math.log2(amax / 448.0))
|
||||
ue8m0_scale = 2.0**exponent
|
||||
# FP8 e4m3 (3-bit mantissa): worst-case error = 16 * scale
|
||||
max_allowed = 16.0 * ue8m0_scale
|
||||
token_diff = diff[t].max().item()
|
||||
assert token_diff <= max_allowed, (
|
||||
f"Token {t} diff {token_diff} exceeds max_allowed "
|
||||
f"{max_allowed} (scale={ue8m0_scale})"
|
||||
)
|
||||
|
||||
|
||||
def test_indexer_gather_accepts_upper_bound_output():
|
||||
"""Gather only exact cu_seq_lens even when dst is over-allocated."""
|
||||
|
||||
head_dim = 128
|
||||
quant_block_size = 128
|
||||
cache_stride = head_dim + head_dim * 4 // quant_block_size
|
||||
valid_tokens = 9
|
||||
upper_bound_tokens = 13
|
||||
block_size = 16
|
||||
num_blocks = 2
|
||||
sentinel = 123
|
||||
device = "cuda"
|
||||
|
||||
k = torch.randn(valid_tokens, head_dim, dtype=torch.bfloat16, device=device)
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
|
||||
)
|
||||
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
|
||||
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
|
||||
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
|
||||
0
|
||||
)
|
||||
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
|
||||
dst_k = torch.full(
|
||||
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
|
||||
)
|
||||
num_scale_bytes = head_dim * 4 // quant_block_size
|
||||
dst_scale = torch.full(
|
||||
(upper_bound_tokens, num_scale_bytes),
|
||||
sentinel,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
ops.cp_gather_indexer_k_quant_cache(
|
||||
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
|
||||
:valid_tokens
|
||||
].view(torch.float32)
|
||||
diff = (k_recovered - k.float()).abs()
|
||||
max_allowed = (16.0 * dst_scale[:valid_tokens].view(torch.float32).max()).item()
|
||||
assert diff.max().item() <= max_allowed
|
||||
assert torch.all(dst_k[valid_tokens:] == sentinel)
|
||||
assert torch.all(dst_scale[valid_tokens:] == sentinel)
|
||||
|
||||
|
||||
# ── Test C: DeepseekV4 attention with values at different magnitudes ───────────
|
||||
|
||||
|
||||
def test_deepseek_v4_quant_magnitude_range():
|
||||
"""Test that quantization handles a range of magnitudes correctly."""
|
||||
|
||||
HEAD_DIM = 512
|
||||
NOPE_DIM = 448
|
||||
HEAD_BYTES = 584
|
||||
block_size = 16
|
||||
num_tokens = 4
|
||||
num_blocks = 2
|
||||
device = "cuda"
|
||||
|
||||
# Create inputs with varying magnitudes: small, medium, large
|
||||
compressed_kv = torch.zeros(
|
||||
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
compressed_kv[0] = 0.001 # very small
|
||||
compressed_kv[1] = 1.0 # unit scale
|
||||
compressed_kv[2] = 100.0 # large
|
||||
compressed_kv[3] = torch.randn(HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
quantize_and_insert_k_cache(
|
||||
compressed_kv, k_cache.view(num_blocks, -1), slot_mapping, block_size
|
||||
)
|
||||
|
||||
out = torch.zeros(1, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
|
||||
0
|
||||
)
|
||||
|
||||
dequantize_and_gather_k_cache(
|
||||
out, k_cache, seq_lens, None, block_table, block_size, offset=0
|
||||
)
|
||||
|
||||
recovered = out[0, :num_tokens]
|
||||
|
||||
# RoPE portion must be exact
|
||||
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs().max()
|
||||
assert rope_diff.item() == 0.0, f"RoPE diff {rope_diff.item()}"
|
||||
|
||||
# NoPE: relative error should be reasonable
|
||||
for t in range(num_tokens):
|
||||
orig = compressed_kv[t, :NOPE_DIM].float()
|
||||
recv = recovered[t, :NOPE_DIM].float()
|
||||
abs_diff = (recv - orig).abs().max().item()
|
||||
magnitude = orig.abs().max().item()
|
||||
if magnitude > 0.01:
|
||||
rel_err = abs_diff / magnitude
|
||||
assert rel_err < 0.15, (
|
||||
f"Token {t}: rel_err={rel_err:.4f}, abs_diff={abs_diff:.6f}, "
|
||||
f"magnitude={magnitude:.4f}"
|
||||
)
|
||||
@@ -0,0 +1,359 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel:
|
||||
|
||||
fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert
|
||||
- Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last 64 dims
|
||||
- KV side: GPT-J RoPE on last 64 + UE8M0 FP8 quant + paged cache insert
|
||||
|
||||
We compare against:
|
||||
- PyTorch reference for RMSNorm + GPT-J RoPE on Q
|
||||
- Existing Triton `quantize_and_insert_k_cache` + round-trip via
|
||||
`dequantize_and_gather_k_cache` for KV
|
||||
|
||||
The kernel is imported via
|
||||
`torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import (
|
||||
dequantize_and_gather_k_cache,
|
||||
quantize_and_insert_k_cache,
|
||||
)
|
||||
|
||||
# ── Constants matching the kernel ────────────────────────────────────────────
|
||||
HEAD_DIM = 512
|
||||
ROPE_DIM = 64
|
||||
NOPE_DIM = HEAD_DIM - ROPE_DIM # 448
|
||||
QUANT_BLOCK = 64
|
||||
FP8_MAX = 448.0
|
||||
HEAD_BYTES = NOPE_DIM + ROPE_DIM * 2 + 8 # 448 + 128 + 8 = 584
|
||||
|
||||
|
||||
# ── PyTorch reference implementations ────────────────────────────────────────
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos: int, rope_dim: int, dtype, device):
|
||||
"""Build a cos||sin cache matching DeepseekV4ScalingRotaryEmbedding layout.
|
||||
cos_sin_cache[pos, :rope_dim/2] = cos(theta), [rope_dim/2:] = sin(theta).
|
||||
"""
|
||||
base = 10000.0
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)
|
||||
)
|
||||
t = torch.arange(max_pos, dtype=torch.float32, device=device)
|
||||
freqs = torch.einsum("i,j -> ij", t, inv_freq) # [max_pos, rope_dim/2]
|
||||
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1) # [max_pos, rope_dim]
|
||||
return cache.to(dtype)
|
||||
|
||||
|
||||
def apply_rope_gptj_last_k(
|
||||
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""GPT-J-style (interleaved-pair) RoPE on the LAST rope_dim elements.
|
||||
|
||||
x: [..., head_dim] float32
|
||||
positions: [num_tokens] int64 (positions[i] corresponds to x[i, ...])
|
||||
cos_sin_cache: [max_pos, rope_dim] float (cos|sin layout)
|
||||
|
||||
Returns rotated x (same shape/dtype).
|
||||
"""
|
||||
rope_dim = cos_sin_cache.shape[-1]
|
||||
half = rope_dim // 2
|
||||
head_dim = x.shape[-1]
|
||||
nope_dim = head_dim - rope_dim
|
||||
|
||||
# Gather cos/sin for each token position: [num_tokens, rope_dim]
|
||||
cs = cos_sin_cache[positions].to(torch.float32) # [N, rope_dim]
|
||||
cos = cs[..., :half] # [N, half]
|
||||
sin = cs[..., half:] # [N, half]
|
||||
|
||||
# Reshape leading dims so we can broadcast: x shape [..., head_dim].
|
||||
# Bring token dim to front; assume x is [num_tokens, ..., head_dim].
|
||||
# We rely on positions being per-token and all other dims sharing the same pos.
|
||||
rope = x[..., nope_dim:].float() # [..., rope_dim]
|
||||
# Make rope pairs: reshape last dim to [half, 2]
|
||||
shape = rope.shape
|
||||
rope = rope.reshape(*shape[:-1], half, 2)
|
||||
even = rope[..., 0] # [..., half]
|
||||
odd = rope[..., 1]
|
||||
|
||||
# Broadcast cos/sin over any heads dim in between. cos/sin are [N, half].
|
||||
# Add singleton dims for intermediate axes.
|
||||
for _ in range(rope.ndim - 3):
|
||||
cos = cos.unsqueeze(1)
|
||||
sin = sin.unsqueeze(1)
|
||||
|
||||
new_even = even * cos - odd * sin
|
||||
new_odd = even * sin + odd * cos
|
||||
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
|
||||
|
||||
out = x.clone().float()
|
||||
out[..., nope_dim:] = rope_rotated
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
def rmsnorm_no_weight(x: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
"""RMSNorm with no learnable weight, matching
|
||||
`RMSNorm(head_dim, has_weight=False)`."""
|
||||
orig_dtype = x.dtype
|
||||
xf = x.float()
|
||||
variance = xf.pow(2).mean(dim=-1, keepdim=True)
|
||||
return (xf * torch.rsqrt(variance + eps)).to(orig_dtype)
|
||||
|
||||
|
||||
# ── Dispatch to the CUDA op (skip test cleanly if it isn't built in) ─────────
|
||||
|
||||
|
||||
def _op_available() -> bool:
|
||||
return hasattr(torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert")
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or not _op_available(),
|
||||
reason="CUDA not available or fused DeepseekV4 op not built in",
|
||||
)
|
||||
|
||||
|
||||
def _call_fused(q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs):
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs
|
||||
)
|
||||
|
||||
|
||||
# ── Test 1: Q path numerical parity ──────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
|
||||
@pytest.mark.parametrize("n_heads", [8, 64])
|
||||
def test_q_path_matches_reference(num_tokens: int, n_heads: int):
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
max_pos = 4096
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
# Reference: RMSNorm (no weight) per head, then GPT-J RoPE on last 64.
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
|
||||
|
||||
# Fused call with dummy KV tensors (KV branch will write slot_mapping=-1 → noop).
|
||||
num_blocks = 2
|
||||
bs = 16
|
||||
kv = torch.zeros(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
k_cache = torch.zeros(
|
||||
num_blocks, bs, HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
).view(num_blocks, -1)
|
||||
slot_mapping = torch.full((num_tokens,), -1, dtype=torch.int64, device=device)
|
||||
q_fused = q.clone()
|
||||
_call_fused(q_fused, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs)
|
||||
|
||||
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
|
||||
# ── Test 2: KV path round-trip byte/value parity ─────────────────────────────
|
||||
|
||||
|
||||
def _ue8m0_per_block_scales(kv_roped_nope_f32: torch.Tensor, qblock: int):
|
||||
"""Return per-token per-block max scale (used to bound FP8 error)."""
|
||||
n_tok, nope = kv_roped_nope_f32.shape
|
||||
n_blocks = nope // qblock
|
||||
blocks = kv_roped_nope_f32.view(n_tok, n_blocks, qblock)
|
||||
absmax = blocks.abs().amax(dim=-1).clamp(min=1e-4)
|
||||
raw = absmax / FP8_MAX
|
||||
exponent = torch.ceil(torch.log2(raw))
|
||||
return torch.pow(2.0, exponent) # [n_tok, n_blocks]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_kv_path_matches_reference(num_tokens: int, block_size: int):
|
||||
torch.manual_seed(1)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
max_pos = 4096
|
||||
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
# ── Reference path: RoPE on kv, then existing Triton quant+insert ──────
|
||||
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
quantize_and_insert_k_cache(
|
||||
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
|
||||
)
|
||||
|
||||
# ── Fused path (dummy q, single head) ──────────────────────────────────
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
q_dummy = torch.zeros(num_tokens, 1, HEAD_DIM, dtype=dtype, device=device)
|
||||
_call_fused(
|
||||
q_dummy,
|
||||
kv,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
# ── Round-trip compare via dequant+gather ──────────────────────────────
|
||||
def _dequant(k_cache_2d):
|
||||
num_reqs = 1
|
||||
max_blocks = num_blocks
|
||||
out = torch.zeros(
|
||||
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
block_table = torch.arange(
|
||||
max_blocks, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
# gather_lens arg is None (use seq_lens)
|
||||
k_cache_3d = k_cache_2d.view(num_blocks, block_size, HEAD_BYTES)
|
||||
dequantize_and_gather_k_cache(
|
||||
out, k_cache_3d, seq_lens, None, block_table, block_size, offset=0
|
||||
)
|
||||
return out[0, :num_tokens]
|
||||
|
||||
recovered_ref = _dequant(k_cache_ref)
|
||||
recovered_fused = _dequant(k_cache_fused)
|
||||
|
||||
# NoPE: per-block UE8M0 FP8 error bound (half-ULP at max = 16 * scale).
|
||||
scales = _ue8m0_per_block_scales(kv_ref[:, :NOPE_DIM].float(), QUANT_BLOCK)
|
||||
for t in range(num_tokens):
|
||||
max_allowed = 16.0 * scales[t].max().item()
|
||||
diff_ref = (
|
||||
(recovered_ref[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
|
||||
)
|
||||
diff_fused = (
|
||||
(recovered_fused[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
|
||||
)
|
||||
assert diff_ref <= max_allowed, (
|
||||
f"ref NoPE token {t} diff {diff_ref} > {max_allowed}"
|
||||
)
|
||||
assert diff_fused <= max_allowed, (
|
||||
f"fused NoPE token {t} diff {diff_fused} > {max_allowed}"
|
||||
)
|
||||
|
||||
# RoPE region: bf16 stored exactly → zero diff.
|
||||
rope_diff = (recovered_fused[:, NOPE_DIM:] - kv_ref[:, NOPE_DIM:]).abs().max()
|
||||
assert rope_diff.item() == 0.0, f"RoPE portion not exact: {rope_diff.item()}"
|
||||
|
||||
# Exact byte equality of the two cache buffers — strong parity.
|
||||
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [4, 17])
|
||||
@pytest.mark.parametrize("pad", [1, 5])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
|
||||
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
|
||||
tokens in the KV branch while still running Q-norm+RoPE on all rows."""
|
||||
torch.manual_seed(3)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
max_pos = 4096
|
||||
total = num_tokens + pad
|
||||
|
||||
kv = torch.randn(total, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(total, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
# Reference: only the first num_tokens kv rows get inserted.
|
||||
kv_ref = apply_rope_gptj_last_k(
|
||||
kv[:num_tokens], positions[:num_tokens], cos_sin_cache
|
||||
)
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
quantize_and_insert_k_cache(
|
||||
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
|
||||
)
|
||||
|
||||
# Fused: pass full-sized q/kv/positions, shorter slot_mapping.
|
||||
q_dummy = torch.zeros(total, 1, HEAD_DIM, dtype=dtype, device=device)
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
_call_fused(
|
||||
q_dummy,
|
||||
kv,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 3: combined single-call Q + KV parity ───────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 17])
|
||||
@pytest.mark.parametrize("n_heads", [8, 64])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
|
||||
torch.manual_seed(2)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
max_pos = 4096
|
||||
|
||||
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
|
||||
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
|
||||
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
# Reference.
|
||||
q_ref = rmsnorm_no_weight(q, eps)
|
||||
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
|
||||
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
|
||||
k_cache_ref = torch.zeros(
|
||||
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
|
||||
)
|
||||
quantize_and_insert_k_cache(
|
||||
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
|
||||
)
|
||||
|
||||
# Fused single call.
|
||||
q_fused = q.clone()
|
||||
k_cache_fused = torch.zeros_like(k_cache_ref)
|
||||
_call_fused(
|
||||
q_fused,
|
||||
kv,
|
||||
k_cache_fused,
|
||||
slot_mapping,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
eps,
|
||||
block_size,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
|
||||
@@ -0,0 +1,98 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for fused_indexer_q_rope_quant.
|
||||
|
||||
Compares the fused Triton kernel against the unfused reference flow used by
|
||||
the DeepseekV4 indexer in model_tracking:
|
||||
q_rot = ops.rotary_embedding(positions, q, None, head_dim, cos_sin_cache,
|
||||
is_neox_style=False,
|
||||
rope_dim_offset=head_dim - rope_dim)
|
||||
q_fp8, q_scale = per_token_group_quant_fp8(q_rot, head_dim, use_ue8m0=True)
|
||||
weights_out = weights * q_scale * softmax_scale * head_scale
|
||||
|
||||
Expects bit-exact equality on both q_fp8 and weights_out.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops.fused_indexer_q import (
|
||||
fused_indexer_q_rope_quant,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
ROPE_DIM = 64
|
||||
N_HEAD = 64
|
||||
MAX_POS = 4096
|
||||
|
||||
|
||||
def _reference(
|
||||
positions: torch.Tensor,
|
||||
q: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
softmax_scale: float,
|
||||
head_scale: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q_rot = q.clone()
|
||||
ops.rotary_embedding(
|
||||
positions,
|
||||
q_rot,
|
||||
None,
|
||||
HEAD_DIM,
|
||||
cos_sin_cache,
|
||||
False, # is_neox_style=False → GPT-J interleaved
|
||||
HEAD_DIM - ROPE_DIM, # rope_dim_offset → rotate the tail
|
||||
False,
|
||||
)
|
||||
q_fp8, q_scale = per_token_group_quant_fp8(
|
||||
q_rot.view(-1, HEAD_DIM).contiguous(),
|
||||
HEAD_DIM,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
q_fp8 = q_fp8.view(-1, N_HEAD, HEAD_DIM)
|
||||
q_scale = q_scale.view(-1, N_HEAD)
|
||||
|
||||
weights_out = weights.to(torch.float32) * q_scale * softmax_scale * head_scale
|
||||
return q_fp8, weights_out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 257])
|
||||
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_fused_indexer_q_rope_quant_matches_unfused(num_tokens, cache_dtype):
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
q = torch.randn(num_tokens, N_HEAD, HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.randint(
|
||||
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=cache_dtype, device=device)
|
||||
weights = torch.randn(num_tokens, N_HEAD, dtype=torch.bfloat16, device=device)
|
||||
softmax_scale = HEAD_DIM**-0.5
|
||||
head_scale = N_HEAD**-0.5
|
||||
|
||||
q_fp8_ref, weights_ref = _reference(
|
||||
positions, q, cos_sin_cache, weights, softmax_scale, head_scale
|
||||
)
|
||||
q_fp8_fused, weights_fused = fused_indexer_q_rope_quant(
|
||||
positions, q.clone(), cos_sin_cache, weights, softmax_scale, head_scale
|
||||
)
|
||||
|
||||
# fp8 tensors aren't directly comparable via torch.equal — reinterpret as int8.
|
||||
ref_bits = q_fp8_ref.view(torch.int8)
|
||||
fused_bits = q_fp8_fused.view(torch.int8)
|
||||
assert torch.equal(ref_bits, fused_bits), (
|
||||
f"q_fp8 mismatch: "
|
||||
f"{(ref_bits != fused_bits).sum().item()} / {ref_bits.numel()} bytes differ"
|
||||
)
|
||||
|
||||
assert torch.equal(weights_ref, weights_fused), (
|
||||
f"weights mismatch: max abs diff "
|
||||
f"{(weights_ref - weights_fused).abs().max().item()}"
|
||||
)
|
||||
@@ -0,0 +1,908 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the fused inverse RoPE + block-scaled FP8 quantization kernel.
|
||||
|
||||
Tests compare the fused kernel against a reference implementation built from
|
||||
the existing separate operations (inverse RoPE via rotate_neox + FP8 quant
|
||||
via per_token_group_quant_fp8).
|
||||
|
||||
The reference faithfully reproduces the exact flow in deepseek_v4_attention.py:295-310:
|
||||
1. Apply inverse RoPE (NeoX style, last rope_dim=64 dims of each head)
|
||||
2. Reshape [T, H, head_dim] -> [T, G, D]
|
||||
3. Transpose+flatten to [G*T, D], quantize, reshape back
|
||||
4. Return o_fp8 and o_scale with strides (D, T*D, 1) and (S, T*S, 1)
|
||||
(non-contiguous [T, G, ...] view backed by contiguous [G, T, ...] memory)
|
||||
|
||||
Usage:
|
||||
pytest tests/kernels/test_fused_inv_rope_fp8_quant.py -v
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.ops.deepseek_v4_ops import fused_inv_rope_fp8_quant
|
||||
|
||||
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
|
||||
HEAD_DIM = 512
|
||||
NOPE_DIM = 448
|
||||
ROPE_DIM = 64
|
||||
QUANT_GROUP_SIZE = 128
|
||||
FP8_MAX = 448.0 # torch.finfo(torch.float8_e4m3fn).max
|
||||
FP8_DTYPE = torch.float8_e4m3fn
|
||||
EPS = 1e-10
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Helpers
|
||||
# =========================================================================
|
||||
|
||||
|
||||
def assert_dequant_close(
|
||||
fp8_a: torch.Tensor,
|
||||
scale_a: torch.Tensor,
|
||||
fp8_b: torch.Tensor,
|
||||
scale_b: torch.Tensor,
|
||||
msg: str = "",
|
||||
):
|
||||
"""Compare two FP8-quantized tensors via their dequantized values.
|
||||
|
||||
Uses cosine-similarity-based diff (same as deep_gemm calc_diff).
|
||||
Both fused and reference paths rotate in fp32 using an fp32
|
||||
cos_sin_cache, so differences are only fp32 ordering ULPs that can
|
||||
occasionally shift FP8 values at quantization boundaries.
|
||||
"""
|
||||
S = scale_a.shape[-1]
|
||||
shape = fp8_a.shape
|
||||
|
||||
dq_a = fp8_a.float() * scale_a.unsqueeze(-1).expand(
|
||||
*shape[:-1], S, QUANT_GROUP_SIZE
|
||||
).reshape(shape)
|
||||
dq_b = fp8_b.float() * scale_b.unsqueeze(-1).expand(
|
||||
*shape[:-1], S, QUANT_GROUP_SIZE
|
||||
).reshape(shape)
|
||||
|
||||
# Cosine diff: 1 - cos_sim (0 = identical, higher = worse)
|
||||
dq_a_flat = dq_a.flatten().float()
|
||||
dq_b_flat = dq_b.flatten().float()
|
||||
cos_sim = torch.nn.functional.cosine_similarity(
|
||||
dq_a_flat.unsqueeze(0), dq_b_flat.unsqueeze(0)
|
||||
).item()
|
||||
diff = 1.0 - cos_sim
|
||||
|
||||
assert diff < 1e-4, f"Dequant diff too large: {diff:.8f} (expected < 1e-4). {msg}"
|
||||
|
||||
|
||||
def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
|
||||
"""GPT-J style rotation: interleaved pairs, negate-swap.
|
||||
|
||||
Matches vllm/model_executor/layers/rotary_embedding/common.py:23-27.
|
||||
DeepseekV4 uses is_neox_style=False, so this is the correct rotation.
|
||||
"""
|
||||
x1 = x[..., ::2]
|
||||
x2 = x[..., 1::2]
|
||||
x = torch.stack((-x2, x1), dim=-1)
|
||||
return x.flatten(-2)
|
||||
|
||||
|
||||
def make_cos_sin_cache(
|
||||
max_pos: int,
|
||||
rope_dim: int = ROPE_DIM,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
device: str = "cuda",
|
||||
) -> torch.Tensor:
|
||||
"""Create a synthetic cos_sin_cache matching the layout used by
|
||||
DeepseekV4ScalingRotaryEmbedding._compute_cos_sin_cache.
|
||||
|
||||
Shape: [max_pos, rope_dim] where first half is cos, second half is sin.
|
||||
The fused kernel requires fp32; callers can override dtype if passing
|
||||
the cache into the bf16-only paths.
|
||||
"""
|
||||
half = rope_dim // 2
|
||||
# Use random but bounded frequencies so cos/sin are well-behaved
|
||||
inv_freq = 1.0 / (
|
||||
10000.0 ** (torch.arange(0, half, device=device, dtype=torch.float32) / half)
|
||||
)
|
||||
t = torch.arange(max_pos, device=device, dtype=torch.float32)
|
||||
freqs = torch.outer(t, inv_freq) # [max_pos, half]
|
||||
cos = freqs.cos()
|
||||
sin = freqs.sin()
|
||||
cache = torch.cat((cos, sin), dim=-1) # [max_pos, rope_dim]
|
||||
return cache.to(dtype)
|
||||
|
||||
|
||||
def reference_inv_rope(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
nope_dim: int = NOPE_DIM,
|
||||
rope_dim: int = ROPE_DIM,
|
||||
) -> torch.Tensor:
|
||||
"""Apply inverse RoPE to the last rope_dim dimensions of each head.
|
||||
|
||||
Matches the GPT-J inverse rotation in pos_encoding_kernels.cu, which
|
||||
promotes the cache to fp32 and performs the rotation in fp32. The
|
||||
result is cast back to the input dtype.
|
||||
|
||||
Args:
|
||||
o: [T, H, head_dim] bf16
|
||||
positions: [T] int64
|
||||
cos_sin_cache: [max_pos, rope_dim] fp32
|
||||
|
||||
Returns:
|
||||
o with inverse RoPE applied on the rope portion (bf16).
|
||||
"""
|
||||
assert cos_sin_cache.dtype == torch.float32
|
||||
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
|
||||
half = rope_dim // 2
|
||||
cos = cos_sin[:, :half]
|
||||
sin = cos_sin[:, half:]
|
||||
|
||||
# GPT-J style: repeat_interleave (not repeat) to match interleaved pairs
|
||||
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(1)
|
||||
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(1)
|
||||
sin = -sin # inverse
|
||||
|
||||
o_pass = o[..., :nope_dim]
|
||||
o_rot_f32 = o[..., nope_dim:].float()
|
||||
o_rot_f32 = o_rot_f32 * cos + rotate_gptj(o_rot_f32) * sin
|
||||
o_rot = o_rot_f32.to(o.dtype)
|
||||
|
||||
return torch.cat([o_pass, o_rot], dim=-1)
|
||||
|
||||
|
||||
def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-block UE8M0 FP8 quantization in pure float32.
|
||||
|
||||
Matches the Triton kernel logic exactly:
|
||||
absmax -> 2^ceil(log2(absmax / fp8_max)) -> clamp(x / scale) -> fp8
|
||||
|
||||
Args:
|
||||
x_f32: [..., quant_group_size] float32 — one or more 128-element blocks.
|
||||
|
||||
Returns:
|
||||
x_fp8: same shape, float8_e4m3fn
|
||||
scales: [...] float32, one scale per block
|
||||
"""
|
||||
assert x_f32.shape[-1] == QUANT_GROUP_SIZE
|
||||
absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS)
|
||||
scale_raw = absmax * (1.0 / FP8_MAX)
|
||||
scale = torch.exp2(torch.ceil(torch.log2(scale_raw)))
|
||||
x_scaled = (x_f32 / scale).clamp(-FP8_MAX, FP8_MAX)
|
||||
x_fp8 = x_scaled.to(FP8_DTYPE)
|
||||
return x_fp8, scale.squeeze(-1)
|
||||
|
||||
|
||||
def reference_inv_rope_fp8_quant(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
n_groups: int,
|
||||
heads_per_group: int,
|
||||
nope_dim: int = NOPE_DIM,
|
||||
rope_dim: int = ROPE_DIM,
|
||||
quant_group_size: int = QUANT_GROUP_SIZE,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Full reference: inverse RoPE in fp32 + UE8M0 FP8 quant in fp32.
|
||||
|
||||
Mimics the Triton kernel's precision path exactly:
|
||||
Load bf16 -> cast to fp32 -> apply inverse RoPE with fp32 cos/sin ->
|
||||
UE8M0 quant in fp32 -> write fp8 + scale
|
||||
|
||||
Returns:
|
||||
o_fp8: [T, G, D] FP8 with strides (D, T*D, 1)
|
||||
o_scale: [T, G, S] FP32 with strides (S, T*S, 1)
|
||||
"""
|
||||
assert cos_sin_cache.dtype == torch.float32
|
||||
T, _H, head_dim = o.shape
|
||||
d = heads_per_group * head_dim
|
||||
S = d // quant_group_size
|
||||
half_rope = rope_dim // 2
|
||||
chunks_per_head = head_dim // quant_group_size
|
||||
|
||||
# Reshape [T, H, head_dim] -> [T, G, heads_per_group, head_dim]
|
||||
o_4d = o.view(T, n_groups, heads_per_group, head_dim)
|
||||
|
||||
# Lookup cos/sin directly in fp32
|
||||
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
|
||||
cos = cos_sin[:, :half_rope] # [T, half_rope] fp32
|
||||
sin = cos_sin[:, half_rope:] # [T, half_rope] fp32
|
||||
|
||||
# Allocate outputs in [G, T, ...] contiguous layout
|
||||
fp8_buf = torch.empty(n_groups, T, d, dtype=FP8_DTYPE, device=o.device)
|
||||
scale_buf = torch.empty(n_groups, T, S, dtype=torch.float32, device=o.device)
|
||||
|
||||
# Process each quant block, matching the Triton kernel's per-program logic
|
||||
for g in range(n_groups):
|
||||
for qb in range(S):
|
||||
head_in_group = qb // chunks_per_head
|
||||
chunk_in_head = qb % chunks_per_head
|
||||
offset = chunk_in_head * quant_group_size
|
||||
|
||||
# Load 128 bf16 elements and promote to fp32 for rotation+quant
|
||||
block = o_4d[:, g, head_in_group, offset : offset + quant_group_size]
|
||||
x = block.float()
|
||||
|
||||
# Apply inverse RoPE in fp32 if this is the last chunk
|
||||
# GPT-J style: interleaved pairs (even=x, odd=y)
|
||||
if chunk_in_head == chunks_per_head - 1:
|
||||
rope_start = nope_dim % quant_group_size # 64
|
||||
rope_region = x[:, rope_start:].clone()
|
||||
x_vals = rope_region[:, ::2]
|
||||
y_vals = rope_region[:, 1::2]
|
||||
x_new = x_vals * cos + y_vals * sin
|
||||
y_new = y_vals * cos - x_vals * sin
|
||||
x = x.clone()
|
||||
x[:, rope_start::2] = x_new
|
||||
x[:, rope_start + 1 :: 2] = y_new
|
||||
|
||||
# UE8M0 quant in fp32
|
||||
x_fp8, scale = _ref_ue8m0_quant_block(x)
|
||||
|
||||
# Write to [G, T, D] contiguous memory
|
||||
fp8_buf[g, :, qb * quant_group_size : (qb + 1) * quant_group_size] = x_fp8
|
||||
scale_buf[g, :, qb] = scale
|
||||
|
||||
# Return transposed views
|
||||
return fp8_buf.transpose(0, 1), scale_buf.transpose(0, 1)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Tests
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,n_groups",
|
||||
[(64, 8), (32, 4), (128, 8)],
|
||||
ids=["H64_G8", "H32_G4", "H128_G8"],
|
||||
)
|
||||
@pytest.mark.parametrize("seed", [0, 42])
|
||||
@torch.inference_mode()
|
||||
def test_correctness(num_tokens, num_heads, n_groups, seed):
|
||||
"""Compare fused kernel against reference for FP8 values and scales."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
# Create inputs
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(
|
||||
max_pos, ROPE_DIM, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
# Reference
|
||||
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Fused kernel
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Check shapes
|
||||
d = heads_per_group * HEAD_DIM
|
||||
S = d // QUANT_GROUP_SIZE
|
||||
assert ref_fp8.shape == (num_tokens, n_groups, d)
|
||||
assert fused_fp8.shape == (num_tokens, n_groups, d)
|
||||
assert ref_scale.shape == (num_tokens, n_groups, S)
|
||||
assert fused_scale.shape == (num_tokens, n_groups, S)
|
||||
|
||||
# Scales: exact match (both use identical UE8M0 algorithm)
|
||||
# Scales may differ by one UE8M0 step (factor of 2) if fp32 rotation
|
||||
# ordering shifts absmax across a power-of-2 boundary. Check ratio is
|
||||
# close to 1.
|
||||
scale_ratio = fused_scale / ref_scale.clamp(min=1e-30)
|
||||
assert scale_ratio.max() <= 2.0 and scale_ratio.min() >= 0.5, (
|
||||
f"Scale ratio out of [0.5, 2]: min={scale_ratio.min():.4f} "
|
||||
f"max={scale_ratio.max():.4f}"
|
||||
)
|
||||
|
||||
# Compare via dequant (Triton vs PyTorch fp32 may differ by ULPs)
|
||||
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,n_groups",
|
||||
[(64, 8), (128, 8)],
|
||||
ids=["H64_G8", "H128_G8"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_output_strides(num_tokens, num_heads, n_groups):
|
||||
"""Verify fused output layout:
|
||||
- FP8: logical [T, G, D] backed by contiguous [G, T, D].
|
||||
- Scale: MN-major TMA-aligned (column-major: T-stride=1).
|
||||
"""
|
||||
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# FP8: logical [T, G, D] backed by [G, T, D] row-major
|
||||
d = heads_per_group * HEAD_DIM
|
||||
expected_fp8_stride = (d, num_tokens * d, 1)
|
||||
assert fused_fp8.stride() == expected_fp8_stride, (
|
||||
f"FP8 stride mismatch: got {fused_fp8.stride()}, expected {expected_fp8_stride}"
|
||||
)
|
||||
|
||||
# Scale: MN-major TMA-aligned layout. After fp8_einsum permutes
|
||||
# [T,G,S] -> [G,T,S], T-dim should have stride 1.
|
||||
# Our output is [T,G,S] = transpose of [G,T,S].
|
||||
# So fused_scale.permute(1,0,2) should have T-stride=1.
|
||||
perm = fused_scale.permute(1, 0, 2) # [G, T, S]
|
||||
assert perm.stride(1) == 1 or num_tokens == 1, (
|
||||
f"Scale T-stride (after permute to [G,T,S]) should be 1, got {perm.stride(1)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
|
||||
@torch.inference_mode()
|
||||
def test_per_group_contiguity(num_tokens):
|
||||
"""FP8 per-group slices must be contiguous. Scale per-group slices
|
||||
are column-major (T-stride=1) — not row-major contiguous, which is
|
||||
correct for TMA loads."""
|
||||
num_heads, n_groups = 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
for g in range(n_groups):
|
||||
fp8_slice = fused_fp8[:, g, :]
|
||||
assert fp8_slice.is_contiguous(), (
|
||||
f"o_fp8[:, {g}, :] is not contiguous: "
|
||||
f"shape={list(fp8_slice.shape)}, stride={list(fp8_slice.stride())}"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_scales_are_power_of_two():
|
||||
"""Verify all scales are exact powers of 2 (UE8M0 property)."""
|
||||
num_tokens, num_heads, n_groups = 32, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
_, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# log2 of a power-of-two is an exact integer
|
||||
log2_scales = torch.log2(fused_scale)
|
||||
residual = (log2_scales - log2_scales.round()).abs()
|
||||
assert residual.max() < 1e-5, (
|
||||
f"Not all scales are powers of 2: max log2 residual = {residual.max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_nope_dims_unchanged():
|
||||
"""Nope dimensions (first 448 per head) should only be quantized,
|
||||
not rotated. Verify by dequantizing and comparing against
|
||||
quantize-only reference (no RoPE)."""
|
||||
num_tokens, num_heads, n_groups = 16, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
# Fused kernel result
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Reference: quantize without RoPE (identity rotation)
|
||||
# Create a zero-sin cache so RoPE is identity
|
||||
zero_cache = torch.zeros_like(cos_sin_cache)
|
||||
half = ROPE_DIM // 2
|
||||
zero_cache[:, :half] = 1.0 # cos = 1
|
||||
# sin = 0 (already zero)
|
||||
|
||||
norope_fp8, norope_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
zero_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Extract nope quant blocks only (first 3 of every 4 blocks per head)
|
||||
chunks_per_head = HEAD_DIM // QUANT_GROUP_SIZE # 4
|
||||
|
||||
for h in range(heads_per_group):
|
||||
for c in range(chunks_per_head - 1): # skip last chunk (has rope)
|
||||
qb = h * chunks_per_head + c
|
||||
start = qb * QUANT_GROUP_SIZE
|
||||
end = start + QUANT_GROUP_SIZE
|
||||
|
||||
fused_nope = fused_fp8[:, :, start:end].view(torch.uint8)
|
||||
norope_nope = norope_fp8[:, :, start:end].view(torch.uint8)
|
||||
assert torch.equal(fused_nope, norope_nope), (
|
||||
f"Nope block (head={h}, chunk={c}) differs between "
|
||||
f"fused and no-rope reference"
|
||||
)
|
||||
|
||||
fused_s = fused_scale[:, :, qb]
|
||||
norope_s = norope_scale[:, :, qb]
|
||||
assert torch.equal(fused_s, norope_s), (
|
||||
f"Nope scale (head={h}, chunk={c}) differs"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_single_token():
|
||||
"""Edge case: single token."""
|
||||
num_tokens, num_heads, n_groups = 1, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.tensor([42], device=device, dtype=torch.long)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_zero_positions():
|
||||
"""Edge case: all positions are 0."""
|
||||
num_tokens, num_heads, n_groups = 16, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.zeros(num_tokens, device=device, dtype=torch.long)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_large_values():
|
||||
"""Edge case: values near FP8 saturation to test clamping."""
|
||||
num_tokens, num_heads, n_groups = 8, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
|
||||
# Create inputs with large values that will saturate FP8
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
o = o * 1000.0 # scale up to force saturation
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_dequant_numerical_accuracy():
|
||||
"""Verify dequantized values are close to the original (after inv RoPE)."""
|
||||
num_tokens, num_heads, n_groups = 32, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
# Get the post-inv-RoPE values (ground truth before quantization)
|
||||
o_after_rope = reference_inv_rope(o.clone(), positions, cos_sin_cache)
|
||||
d = heads_per_group * HEAD_DIM
|
||||
o_after_rope = o_after_rope.view(num_tokens, n_groups, d)
|
||||
|
||||
# Get fused quantized output
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Dequantize: broadcast scale [T, G, S] to [T, G, D] via repeat
|
||||
S = d // QUANT_GROUP_SIZE
|
||||
scale_expanded = (
|
||||
fused_scale.unsqueeze(-1)
|
||||
.expand(num_tokens, n_groups, S, QUANT_GROUP_SIZE)
|
||||
.reshape(num_tokens, n_groups, d)
|
||||
)
|
||||
dequant = fused_fp8.float() * scale_expanded
|
||||
|
||||
# Check relative error.
|
||||
# FP8 e4m3 with UE8M0 (power-of-two scales that round UP) quantizes more
|
||||
# coarsely than optimal scaling. Both paths rotate in fp32, so the bulk
|
||||
# of the error comes from UE8M0 quantization itself (~10-12% typical).
|
||||
o_gt = o_after_rope.transpose(0, 1).contiguous().transpose(0, 1)
|
||||
dequant_contig = dequant.transpose(0, 1).contiguous().transpose(0, 1)
|
||||
|
||||
abs_err = (dequant_contig.float() - o_gt.float()).abs()
|
||||
rel_err = abs_err / (o_gt.float().abs().clamp(min=1e-6))
|
||||
mean_rel_err = rel_err.mean().item()
|
||||
|
||||
assert mean_rel_err < 0.15, (
|
||||
f"Mean relative error too high: {mean_rel_err:.4f} (expected < 0.15)"
|
||||
)
|
||||
|
||||
|
||||
def _unfused_inv_rope_fp8_quant(
|
||||
o: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
n_groups: int,
|
||||
heads_per_group: int,
|
||||
nope_dim: int = NOPE_DIM,
|
||||
rope_dim: int = ROPE_DIM,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Unfused path matching deepseek_v4_attention.py:295-310.
|
||||
|
||||
Uses the production CUDA RoPE kernel + per_token_group_quant_fp8.
|
||||
"""
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
|
||||
head_dim = o.shape[-1]
|
||||
rope_dim_offset = head_dim - rope_dim
|
||||
|
||||
# Step 1: In-place CUDA RoPE (same as production)
|
||||
ops.rotary_embedding(
|
||||
positions,
|
||||
o,
|
||||
None,
|
||||
head_dim,
|
||||
cos_sin_cache,
|
||||
False, # is_neox=False for DeepseekV4 (GPT-J style)
|
||||
rope_dim_offset=rope_dim_offset,
|
||||
inverse=True,
|
||||
)
|
||||
|
||||
# Step 2: Reshape + quant + reshape (same as production)
|
||||
T = o.shape[0]
|
||||
d = heads_per_group * head_dim
|
||||
o = o.view(T, n_groups, -1)
|
||||
o_flat = o.transpose(0, 1).contiguous().reshape(-1, d)
|
||||
o_fp8, o_scale = per_token_group_quant_fp8(
|
||||
o_flat,
|
||||
group_size=QUANT_GROUP_SIZE,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
o_fp8 = o_fp8.view(n_groups, T, d).transpose(0, 1)
|
||||
o_scale = o_scale.view(n_groups, T, -1).transpose(0, 1)
|
||||
return o_fp8, o_scale
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# End-to-end test including fp8_einsum
|
||||
# =========================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128, 1024])
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,n_groups",
|
||||
[(64, 8)],
|
||||
ids=["H64_G8"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
|
||||
"""End-to-end: fused inv_rope+quant → fp8_einsum must match
|
||||
unfused CUDA_rope+quant → fp8_einsum bitwise.
|
||||
|
||||
This catches stride/layout bugs that only manifest when the einsum
|
||||
kernel actually consumes the quantized activations.
|
||||
"""
|
||||
from deep_gemm.utils.math import ceil_div
|
||||
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_einsum,
|
||||
per_block_cast_to_fp8,
|
||||
transform_sf_into_required_layout,
|
||||
)
|
||||
|
||||
heads_per_group = num_heads // n_groups
|
||||
d = heads_per_group * HEAD_DIM
|
||||
o_lora_rank = 1024
|
||||
max_pos = 4096
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), device=device, dtype=torch.long
|
||||
)
|
||||
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
|
||||
|
||||
# -- Weight quantization (shared between both paths) --
|
||||
w = torch.randn(n_groups, o_lora_rank, d, device=device, dtype=torch.bfloat16)
|
||||
w_fp8 = torch.empty_like(w, dtype=torch.float8_e4m3fn)
|
||||
w_scale = torch.empty(
|
||||
n_groups,
|
||||
ceil_div(o_lora_rank, 128),
|
||||
ceil_div(d, 128),
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
for g in range(n_groups):
|
||||
w_fp8[g], w_scale[g] = per_block_cast_to_fp8(w[g], use_ue8m0=True)
|
||||
|
||||
recipe = (1, 1, 128)
|
||||
w_scale_t = transform_sf_into_required_layout(
|
||||
sf=w_scale,
|
||||
mn=o_lora_rank,
|
||||
k=d,
|
||||
recipe=(1, 128, 128),
|
||||
num_groups=n_groups,
|
||||
is_sfa=False,
|
||||
)
|
||||
|
||||
# -- UNFUSED path --
|
||||
ref_fp8, ref_scale = _unfused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
z_ref = torch.empty(
|
||||
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd", (ref_fp8, ref_scale), (w_fp8, w_scale_t), z_ref, recipe=recipe
|
||||
)
|
||||
|
||||
# -- FUSED path --
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
z_fused = torch.empty(
|
||||
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(fused_fp8, fused_scale),
|
||||
(w_fp8, w_scale_t),
|
||||
z_fused,
|
||||
recipe=recipe,
|
||||
)
|
||||
|
||||
# -- Checks --
|
||||
# Einsum output: Triton and CUDA both rotate in fp32 now, so diffs
|
||||
# come from fp32 ordering and UE8M0 boundary shifts only.
|
||||
# Use relative diff (same metric as test_fp8_einsum.py).
|
||||
from deep_gemm.testing import calc_diff
|
||||
|
||||
z_diff = calc_diff(z_fused, z_ref)
|
||||
assert z_diff < 0.01, (
|
||||
f"Einsum output diff too large: {z_diff:.6f} (expected < 0.01)"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32, 256])
|
||||
@torch.inference_mode()
|
||||
def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config):
|
||||
"""Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style,
|
||||
mscale=0, YaRN scaling) matching the production config."""
|
||||
|
||||
num_heads = 64
|
||||
n_groups = 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
device = "cuda"
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Build YaRN-scaled cos_sin_cache matching real DeepSeek V3/V4 config
|
||||
# (mscale=0 → mscale=1.0, so no magnitude scaling)
|
||||
from vllm.model_executor.layers.rotary_embedding.common import (
|
||||
yarn_find_correction_range,
|
||||
yarn_linear_ramp_mask,
|
||||
)
|
||||
|
||||
scaling_factor = 16
|
||||
base = 10000.0
|
||||
max_pos = 65536
|
||||
beta_fast, beta_slow = 32, 1
|
||||
|
||||
pos_freqs = base ** (
|
||||
torch.arange(0, ROPE_DIM, 2, dtype=torch.float32, device=device) / ROPE_DIM
|
||||
)
|
||||
inv_freq_extra = 1.0 / pos_freqs
|
||||
inv_freq_interp = 1.0 / (scaling_factor * pos_freqs)
|
||||
low, high = yarn_find_correction_range(
|
||||
beta_fast, beta_slow, ROPE_DIM, base, max_pos
|
||||
)
|
||||
mask = 1 - yarn_linear_ramp_mask(low, high, ROPE_DIM // 2, dtype=torch.float32).to(
|
||||
device
|
||||
)
|
||||
inv_freq = inv_freq_interp * (1 - mask) + inv_freq_extra * mask
|
||||
t = torch.arange(max_pos * scaling_factor, device=device, dtype=torch.float32)
|
||||
freqs = torch.outer(t, inv_freq)
|
||||
# mscale=0 → yarn_get_mscale returns 1.0
|
||||
cos_sin_cache = torch.cat([freqs.cos(), freqs.sin()], dim=-1) # fp32
|
||||
|
||||
o = torch.randn(
|
||||
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
positions = torch.randint(0, 4096, (num_tokens,), device=device, dtype=torch.long)
|
||||
|
||||
# UNFUSED: CUDA RoPE with is_neox=False (GPT-J)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
per_token_group_quant_fp8,
|
||||
)
|
||||
|
||||
o_unfused = o.clone()
|
||||
ops.rotary_embedding(
|
||||
positions,
|
||||
o_unfused,
|
||||
None,
|
||||
HEAD_DIM,
|
||||
cos_sin_cache,
|
||||
False, # is_neox=False (GPT-J style)
|
||||
rope_dim_offset=NOPE_DIM,
|
||||
inverse=True,
|
||||
)
|
||||
d = heads_per_group * HEAD_DIM
|
||||
T = num_tokens
|
||||
o_unfused = o_unfused.view(T, n_groups, d)
|
||||
o_flat = o_unfused.transpose(0, 1).contiguous().reshape(-1, d)
|
||||
ref_fp8, ref_scale = per_token_group_quant_fp8(
|
||||
o_flat,
|
||||
group_size=QUANT_GROUP_SIZE,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
ref_fp8 = ref_fp8.view(n_groups, T, d).transpose(0, 1)
|
||||
ref_scale = ref_scale.view(n_groups, T, -1).transpose(0, 1)
|
||||
|
||||
# FUSED: use the real YaRN-scaled cos_sin_cache
|
||||
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
|
||||
o.clone(),
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
n_groups,
|
||||
heads_per_group,
|
||||
)
|
||||
|
||||
# Scales must match exactly (same UE8M0 algorithm)
|
||||
# Compare via dequant (Triton bf16 rotation may differ from CUDA by 1 ULP)
|
||||
assert_dequant_close(
|
||||
ref_fp8, ref_scale, fused_fp8, fused_scale, msg="Real DeepSeek V4 rope"
|
||||
)
|
||||
@@ -718,7 +718,6 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [2000, 6000, 30000, 80000],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="mixed_all_paths",
|
||||
@@ -727,7 +726,6 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [2048, 4096, 8192, 16000],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="all_decode_medium",
|
||||
@@ -736,7 +734,6 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [70000, 100000, 163840],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="all_large",
|
||||
@@ -745,7 +742,6 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [32767, 32768, 32769, 32772],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="large_threshold_boundary",
|
||||
@@ -754,7 +750,6 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [5000],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="single_row_medium",
|
||||
@@ -772,15 +767,15 @@ def test_persistent_topk_stress() -> None:
|
||||
pytest.param(
|
||||
{
|
||||
"seq_lens": [100, 2048, 10000, 80000],
|
||||
"top_k": 2048,
|
||||
"data_type": "random",
|
||||
},
|
||||
id="trivial_medium_large_mix",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("top_k", [512, 2048])
|
||||
@torch.inference_mode()
|
||||
def test_persistent_topk(test_config: dict) -> None:
|
||||
def test_persistent_topk(test_config: dict, top_k: int) -> None:
|
||||
"""
|
||||
Tests specific to the persistent_topk kernel:
|
||||
- Mixed medium/large rows in the same batch (dynamic per-row dispatch)
|
||||
@@ -790,14 +785,15 @@ def test_persistent_topk(test_config: dict) -> None:
|
||||
run_large_context_topk_test(
|
||||
batch_size=len(test_config["seq_lens"]),
|
||||
seq_lens=test_config["seq_lens"],
|
||||
top_k=test_config["top_k"],
|
||||
top_k=top_k,
|
||||
data_type=test_config.get("data_type", "random"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@pytest.mark.parametrize("top_k", [512, 2048])
|
||||
@torch.inference_mode()
|
||||
def test_persistent_topk_padded_stride() -> None:
|
||||
def test_persistent_topk_padded_stride(top_k: int) -> None:
|
||||
"""
|
||||
Test persistent_topk with padded logits (large stride, small seq_len)
|
||||
to simulate the e2e CUDAGraph scenario where fp8_paged_mqa_logits
|
||||
@@ -806,7 +802,6 @@ def test_persistent_topk_padded_stride() -> None:
|
||||
set_random_seed(42)
|
||||
torch.set_default_device("cuda:0")
|
||||
|
||||
top_k = 2048
|
||||
batch_size = 4
|
||||
padded_stride = 163840 # DeepSeek-V3.2 max_model_len
|
||||
actual_seq_lens = [3000, 5000, 8000, 12000]
|
||||
|
||||
Reference in New Issue
Block a user