[Feat] DeepSeek V4 Rebased (#40860)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: qizixi <zixi@inferact.ai>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <yongye@inferact.ai>
Co-authored-by: Simon Mo <simon@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Roy Wang <yasong.wang@inferact.ai>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: youkaichao <youkaichao@gmail.com>
Co-authored-by: Zhewen Li <jerven.vllm@gmail.com>
Co-authored-by: Zijing Liu <liuzijing2014@gmail.com>
Co-authored-by: khluu <khluu000@gmail.com>
Co-authored-by: qizixi <zixi@inferact.ai>
Co-authored-by: Zhewen Li <zhewenli@inferact.ai>
This commit is contained in:
Yifan Qiao
2026-04-26 18:31:08 -07:00
committed by GitHub
co-authored by Yongye Zhu Yongye Zhu Simon Mo Bugen Zhao Giancarlo Delfin Jee Jee Li Nick Hill Roger Wang Roy Wang Woosuk Kwon youkaichao Zhewen Li Zijing Liu khluu qizixi Zhewen Li
parent 32e45636e3
commit 4d51588e23
150 changed files with 16312 additions and 716 deletions
+5
View File
@@ -116,6 +116,11 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):
model_kwargs["attention_config"] = {"backend": attn_backend.backend.name}
model_kwargs["tensor_parallel_size"] = tp_size
# Cap warmup memory: tests use small max_model_len (1024) but the
# engine default max_num_batched_tokens is 16384. Warming up large
# models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM.
model_kwargs.setdefault("max_num_batched_tokens", 8192)
# Sparse MLA models (DSv3.2) hit an over-strict inductor assertion in
# decompose_auto_functionalized when +rotary_embedding is forced into
# the compile graph. Disable qk_norm+rope fusion (which auto-enables
@@ -9,8 +9,8 @@ from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
calc_diff,
fp8_mqa_logits,
fp8_paged_mqa_logits,
fp8_fp4_mqa_logits,
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
)
@@ -127,8 +127,8 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
logits = fp8_mqa_logits(
q_fp8, kv_fp8, weights, ks, ke, clean_logits=clean_logits
logits = fp8_fp4_mqa_logits(
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
)
ref_logits = _ref_fp8_mqa_logits(
@@ -150,7 +150,7 @@ def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_paged_mqa_logits(
def _ref_fp8_fp4_paged_mqa_logits(
q: torch.Tensor,
kv_cache: torch.Tensor,
weights: torch.Tensor,
@@ -205,8 +205,10 @@ def _ref_fp8_paged_mqa_logits(
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("clean_logits", [True, False])
def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
def test_deepgemm_fp8_fp4_paged_mqa_logits():
# NOTE: clean_logits=True is incompatible with the 2D context_lens
# required by csrc/apis/attention.hpp; only the False path is exercised.
clean_logits = False
torch.manual_seed(0)
random.seed(0)
@@ -258,21 +260,29 @@ def test_deepgemm_fp8_paged_mqa_logits(clean_logits: bool):
q_fp8 = q.to(torch.float8_e4m3fn)
kv_cache_fp8 = kv_cache_cast_to_fp8(kv_cache)
# deep_gemm paged MQA logits requires 2D context_lens of
# shape (B, next_n) (csrc/apis/attention.hpp:332-335);
# see indexer.py:607-608. For each batch/next_n token, the
# effective context length is context_lens[b] - next_n + j + 1.
next_n_arange = torch.arange(next_n, device="cuda", dtype=torch.int32)
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
schedule_metadata = get_paged_mqa_logits_metadata(
context_lens, blocksize, get_num_sms()
context_lens_2d, blocksize, get_num_sms()
)
logits = fp8_paged_mqa_logits(
q_fp8,
logits = fp8_fp4_paged_mqa_logits(
(q_fp8, None),
kv_cache_fp8,
weights,
context_lens,
context_lens_2d,
block_tables,
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
)
ref_logits = _ref_fp8_paged_mqa_logits(
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
weights,
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness + large-token-count launch tests for fused_q_kv_rmsnorm.
Before the grid-dim fix the kernel used grid ``(2, num_tokens)``, which hit
CUDA's 65535 grid-y cap for ``num_tokens >= 65536`` and failed with
``Triton Error [CUDA]: invalid argument`` at every large chunked-prefill
profile run. These tests pin the new grid layout.
"""
from __future__ import annotations
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.attention.ops.deepseek_v4_ops import fused_q_kv_rmsnorm
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fused_q_kv_rmsnorm requires a CUDA/ROCm device",
)
def _ref_rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
x_f32 = x.to(torch.float32)
variance = x_f32.pow(2).mean(dim=-1, keepdim=True)
y = x_f32 * torch.rsqrt(variance + eps) * w.to(torch.float32)
return y.to(x.dtype)
@pytest.mark.parametrize("num_tokens", [1, 17, 1024, 8192])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_fused_q_kv_rmsnorm_correctness(num_tokens: int, dtype: torch.dtype):
torch.manual_seed(0)
device = "cuda"
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
eps = 1e-6
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, eps)
qr_ref = _ref_rmsnorm(qr, qw, eps)
kv_ref = _ref_rmsnorm(kv, kvw, eps)
tol = dict(rtol=1e-2, atol=1e-2)
torch.testing.assert_close(qr_out, qr_ref, **tol)
torch.testing.assert_close(kv_out, kv_ref, **tol)
@pytest.mark.parametrize("num_tokens", [65535, 65536, 131072])
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
qr = torch.randn(num_tokens, q_size, dtype=dtype, device=device)
kv = torch.randn(num_tokens, kv_size, dtype=dtype, device=device)
qw = torch.randn(q_size, dtype=dtype, device=device)
kvw = torch.randn(kv_size, dtype=dtype, device=device)
qr_out, kv_out = fused_q_kv_rmsnorm(qr, kv, qw, kvw, 1e-6)
# spot-check a couple of rows against the torch reference
for row in (0, num_tokens // 2, num_tokens - 1):
torch.testing.assert_close(
qr_out[row],
_ref_rmsnorm(qr[row : row + 1], qw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
torch.testing.assert_close(
kv_out[row],
_ref_rmsnorm(kv[row : row + 1], kvw, 1e-6)[0],
rtol=1e-2,
atol=1e-2,
)
+195 -1
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit-test DeepGEMM FP8 kernels (no DeepEP).
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
@@ -21,6 +21,8 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
FusedMoEQuantDesc,
fp8_w8a8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
@@ -204,3 +206,195 @@ def test_deepgemm_vs_triton(m, n, k, topk, num_experts, monkeypatch, workspace_i
f"DeepGEMM path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
# ---------------------------------------------------------------------------
# FP4 weight tests (DeepGEMM m_grouped_fp8_fp4_gemm_nt_contiguous)
# ---------------------------------------------------------------------------
def make_mxfp4_weights(
e: int,
n: int,
k: int,
):
"""
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
w2 shape: (E, K, N//2) uint8 — packed FP4
w1_s shape: (E, 2N, K//32) float32 — per-row block-32 scales
w2_s shape: (E, K, N//32) float32 — per-row block-32 scales
w1_bf16: (E, 2N, K) — original BF16 for reference
w2_bf16: (E, K, N) — original BF16 for reference
"""
from deep_gemm.utils.math import per_token_cast_to_fp4
dtype = torch.bfloat16
gran_k = 32 # MXFP4 block size
# bf16 reference weights — scale by 1/sqrt(dim) for numerical stability
w1_bf16 = torch.randn(e, 2 * n, k, device="cuda", dtype=dtype) * (k**-0.5)
w2_bf16 = torch.randn(e, k, n, device="cuda", dtype=dtype) * (n**-0.5)
# Quantize per-expert to FP4
w1 = torch.empty(e, 2 * n, k // 2, device="cuda", dtype=torch.uint8)
w2 = torch.empty(e, k, n // 2, device="cuda", dtype=torch.uint8)
w1_s = torch.empty(
e, 2 * n, math.ceil(k / gran_k), device="cuda", dtype=torch.float32
)
w2_s = torch.empty(e, k, math.ceil(n / gran_k), device="cuda", dtype=torch.float32)
for i in range(e):
w1[i], w1_s[i] = per_token_cast_to_fp4(
w1_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
w2[i], w2_s[i] = per_token_cast_to_fp4(
w2_bf16[i].float(), use_ue8m0=True, gran_k=gran_k
)
return w1, w2, w1_s, w2_s, w1_bf16, w2_bf16
def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
"""BF16 token-loop MoE reference for correctness testing."""
import torch.nn.functional as F
num_tokens, hidden_size = x.shape
intermediate = w1.shape[1] // 2
top_k = topk_ids.shape[1]
output = torch.zeros(num_tokens, hidden_size, dtype=torch.float32, device=x.device)
for t in range(num_tokens):
for kk in range(top_k):
e = topk_ids[t, kk].item()
w = topk_weights[t, kk].item()
fc1 = x[t : t + 1].float() @ w1[e].float().T
linear = fc1[:, :intermediate]
gate = fc1[:, intermediate:]
act = F.silu(gate) * linear
fc2 = act @ w2[e].float().T
output[t] += w * fc2[0]
return output.to(torch.bfloat16)
def run_single_fp4_case(m, n, k, topk, num_experts):
"""
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
# FP4 expert weight tensors + BF16 originals for reference
w1, w2, w1_s, w2_s, w1_bf16, w2_bf16 = make_mxfp4_weights(num_experts, n, k)
router_logits = torch.randn(m, num_experts, device="cuda", dtype=torch.float32)
topk_weights, topk_ids = torch.topk(router_logits, k=topk, dim=-1)
topk_weights = torch.nn.functional.softmax(topk_weights, dim=-1)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
)
from vllm.platforms import current_platform
_fp8_dtype = current_platform.fp8_dtype()
_block_shape = GroupShape(128, 128)
quant_config = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_a2=FusedMoEQuantDesc(_fp8_dtype, _block_shape, None, None, None, None),
_w1=FusedMoEQuantDesc("mxfp4", None, w1_s, None, None, None),
_w2=FusedMoEQuantDesc("mxfp4", None, w2_s, None, None, None),
)
moe_config = make_dummy_moe_config()
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
DeepGemmFP4Experts,
)
deep_gemm_fp4_experts = mk.FusedMoEKernel(
prepare_finalize=maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
fused_experts=DeepGemmFP4Experts(
moe_config=moe_config,
quant_config=quant_config,
),
inplace=False,
)
# DeepGEMM FP4 path
out_deepgemm_fp4 = deep_gemm_fp4_experts.apply(
hidden_states=tokens_bf16,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
global_num_experts=num_experts,
activation=MoEActivation.SILU,
apply_router_weight_on_input=False,
expert_map=None,
)
# BF16 reference using the same original weights
out_ref = _bf16_moe_reference(tokens_bf16, w1_bf16, w2_bf16, topk_weights, topk_ids)
# FP4 vs BF16 reference: quantization error from FP4 weights + FP8 activations
diff = calc_diff(out_deepgemm_fp4, out_ref)
assert diff < 0.05, f"FP4 diff exceeded 5%: {diff}"
# DeepSeek V4 dims: H=4096, I=2048, so N=2*I=4096, K=H=4096.
# FP4 quantization with block_k=32 needs large K for good accuracy.
FP4_MNKs = [
(128, 4096, 4096), # DeepSeek V4 shape
(256, 2048, 2048), # Half-size variant
]
FP4_TOPKS = [2]
FP4_NUM_EXPERTS = [8]
@pytest.mark.parametrize(("m", "n", "k"), FP4_MNKs)
@pytest.mark.parametrize("topk", FP4_TOPKS)
@pytest.mark.parametrize("num_experts", FP4_NUM_EXPERTS)
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_fp4_vs_triton(
m, n, k, topk, num_experts, monkeypatch, workspace_init
):
pytest.importorskip("deep_gemm.utils.math")
with monkeypatch.context() as mp:
mp.setenv("VLLM_USE_DEEP_GEMM", "1")
_DeepGemmFP4Experts = importlib.import_module(
"vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe"
).DeepGemmFP4Experts
call_counter = {"cnt": 0}
orig_fn = _DeepGemmFP4Experts.apply
def _spy_apply(*args, **kwargs):
call_counter["cnt"] += 1
return orig_fn(*args, **kwargs)
monkeypatch.setattr(_DeepGemmFP4Experts, "apply", _spy_apply)
if topk > num_experts:
pytest.skip(f"topk={topk} > num_experts={num_experts}")
run_single_fp4_case(
m=m,
n=n,
k=k,
topk=topk,
num_experts=num_experts,
)
# ensure that the DeepGEMM FP4 path was indeed taken.
assert call_counter["cnt"] == 1, (
f"DeepGEMM FP4 path was not executed during the test. "
f"Call counter: {call_counter['cnt']}"
)
@@ -0,0 +1,186 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
fused_topk_bias,
)
from vllm.platforms import current_platform
def _torch_topk_softplus_sqrt(
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
e_score_correction_bias: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
scores = F.softplus(gating_output.float()).sqrt()
original_scores = scores
if e_score_correction_bias is not None:
scores_for_choice = scores + e_score_correction_bias.unsqueeze(0)
else:
scores_for_choice = scores
if hash_indices_table is not None:
assert input_ids is not None
topk_ids = hash_indices_table[input_ids.long()]
else:
topk_ids = torch.topk(scores_for_choice, k=topk, dim=-1, sorted=True)[1]
topk_weights = original_scores.gather(1, topk_ids.long())
if renormalize:
topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
if routed_scaling_factor != 1.0:
topk_weights = topk_weights * routed_scaling_factor
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
def test_sqrtsoftplus_bias_uses_deepseek_v4_routing_method():
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=True,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.DeepseekV4
)
assert (
get_routing_method_type(
scoring_func="sqrtsoftplus",
top_k=8,
renormalize=False,
num_expert_group=None,
has_e_score_bias=True,
)
== RoutingMethodType.Unspecified
)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [128, 256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 1.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
e_score_correction_bias = torch.randn(
(num_experts,), dtype=torch.float32, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
e_score_correction_bias=e_score_correction_bias,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=e_score_correction_bias,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
)
# Different kernels may return the topk experts in different orders when
# scores tie; sort by expert id before comparing.
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform."
)
@pytest.mark.parametrize("num_tokens", [1, 33, 128])
@pytest.mark.parametrize("hidden_size", [1024, 2048])
@pytest.mark.parametrize("num_experts", [256, 384, 512])
@pytest.mark.parametrize("topk", [6, 8, 16])
@pytest.mark.parametrize("renormalize", [True, False])
@pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32])
def test_fused_topk_softplus_sqrt_hash(
num_tokens: int,
hidden_size: int,
num_experts: int,
topk: int,
renormalize: bool,
routed_scaling_factor: float,
dtype: torch.dtype,
):
torch.manual_seed(0)
vocab_size = 1024
hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda")
gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda")
# Per-token fixed expert selection: for each vocab id pick `topk` distinct
# experts.
hash_indices_table = torch.stack(
[torch.randperm(num_experts)[:topk] for _ in range(vocab_size)]
).to(device="cuda", dtype=torch.int32)
input_ids = torch.randint(
0, vocab_size, (num_tokens,), dtype=torch.int32, device="cuda"
)
topk_weights_ref, topk_ids_ref = _torch_topk_softplus_sqrt(
gating_output=gating_output,
topk=topk,
renormalize=renormalize,
routed_scaling_factor=routed_scaling_factor,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
topk_weights, topk_ids = fused_topk_bias(
hidden_states=hidden_states,
gating_output=gating_output,
scoring_func="sqrtsoftplus",
e_score_correction_bias=None,
topk=topk,
renormalize=renormalize,
input_tokens=input_ids,
hash_indices_table=hash_indices_table,
routed_scaling_factor=routed_scaling_factor,
)
sorted_ref_ids, idx_ref = topk_ids_ref.sort(dim=-1)
sorted_ids, idx_ops = topk_ids.sort(dim=-1)
torch.testing.assert_close(sorted_ref_ids, sorted_ids, atol=0, rtol=0)
sorted_w_ref = topk_weights_ref.gather(1, idx_ref)
sorted_w = topk_weights.gather(1, idx_ops)
torch.testing.assert_close(sorted_w_ref, sorted_w, atol=2e-2, rtol=1e-2)
+311
View File
@@ -0,0 +1,311 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Round-trip tests for compressor → FP8 quant + KV cache insert → gather + dequant.
Two paths tested:
A) DeepseekV4 Attention: head_dim=512 (448 FP8 nope + 64 bf16 rope), quant_block=64
B) Indexer: head_dim=128 (all FP8), quant_block=128
These serve as golden references for validating the future fused
compressor+quant+cache kernel.
"""
import math
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.v1.attention.ops.deepseek_v4_ops import (
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
def _ue8m0_reference(x: torch.Tensor, block_size: int, fp8_max: float):
"""PyTorch reference for UE8M0 FP8 quantization (per-block, power-of-2 scale).
Returns (x_fp8, scales) where x_fp8 is float8_e4m3fn and scales are float32.
"""
assert x.dim() == 1
n = x.numel()
n_blocks = math.ceil(n / block_size)
x_fp8 = torch.zeros(n, dtype=torch.float8_e4m3fn, device=x.device)
scales = torch.zeros(n_blocks, dtype=torch.float32, device=x.device)
for i in range(n_blocks):
start = i * block_size
end = min(start + block_size, n)
block = x[start:end].float()
amax = block.abs().max().clamp(min=1e-4)
raw_scale = amax / fp8_max
exponent = math.ceil(math.log2(raw_scale.item()))
scale = 2.0**exponent
scales[i] = scale
quantized = (block / scale).clamp(-fp8_max, fp8_max)
x_fp8[start:end] = quantized.to(torch.float8_e4m3fn)
return x_fp8, scales
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache
→ compare against original."""
HEAD_DIM = 512
NOPE_DIM = 448
HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale
FP8_MAX = 448.0
QUANT_BLOCK = 64
num_blocks = (num_tokens + block_size - 1) // block_size + 1
device = "cuda"
# Random compressed_kv (simulates compressor output)
compressed_kv = torch.randn(
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
# ── Quant + insert ──────────────────────────────────────────────────
k_cache = torch.zeros(
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
)
k_cache_2d = k_cache.view(num_blocks, -1)
# Sequential slot mapping: token i → slot i
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(
compressed_kv, k_cache_2d, slot_mapping, block_size=block_size
)
# ── Gather + dequant ────────────────────────────────────────────────
num_reqs = 1
max_blocks_per_seq = num_blocks
out = torch.zeros(
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
# block_table: request 0 uses physical blocks 0, 1, ...
block_table = torch.arange(
max_blocks_per_seq, dtype=torch.int32, device=device
).unsqueeze(0)
dequantize_and_gather_k_cache(
out, k_cache, seq_lens, None, block_table, block_size, offset=0
)
recovered = out[0, :num_tokens]
# ── NoPE portion (first 448): FP8 quantized, expect UE8M0 error ──
nope_orig = compressed_kv[:, :NOPE_DIM].float()
nope_recv = recovered[:, :NOPE_DIM].float()
nope_diff = (nope_recv - nope_orig).abs()
# Per-token check: FP8 e4m3 (3-bit mantissa) worst-case error is
# half-ULP at the largest representable value. At y ≈ 448 (max),
# ULP = 2^(8-3) = 32, so error ≤ 16 * scale.
for t in range(num_tokens):
_, scales = _ue8m0_reference(
compressed_kv[t, :NOPE_DIM].float(), QUANT_BLOCK, FP8_MAX
)
max_allowed = 16.0 * scales.max().item()
token_diff = nope_diff[t].max().item()
assert token_diff <= max_allowed, (
f"Token {t} nope diff {token_diff} exceeds max_allowed "
f"{max_allowed} (scale={scales.max().item()})"
)
# ── RoPE portion (last 64): stored as bf16, should be exact ─────
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs()
assert rope_diff.max().item() == 0.0, (
f"RoPE portion should be exact but got max diff {rope_diff.max().item()}"
)
# ── Test B: Indexer path ────────────────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""k → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
→ manual dequant → compare against original."""
HEAD_DIM = 128
QUANT_BLOCK_SIZE = 128
# cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132
CACHE_STRIDE = HEAD_DIM + HEAD_DIM * 4 // QUANT_BLOCK_SIZE
num_blocks = (num_tokens + block_size - 1) // block_size + 1
device = "cuda"
# Random K (simulates compressor output for indexer)
k = torch.randn(num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
# ── Quant + insert ──────────────────────────────────────────────────
kv_cache = torch.zeros(
num_blocks, block_size, CACHE_STRIDE, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
# ── Gather ──────────────────────────────────────────────────────────
max_blocks_per_seq = num_blocks
block_table = torch.arange(
max_blocks_per_seq, dtype=torch.int32, device=device
).unsqueeze(0)
cu_seq_lens = torch.tensor([0, num_tokens], dtype=torch.int32, device=device)
# dst_k: [total_seq_len, head_dim] as uint8 (raw FP8 bytes)
dst_k = torch.zeros(num_tokens, HEAD_DIM, dtype=torch.uint8, device=device)
# dst_scale: [total_seq_len, head_dim/quant_block*4] as uint8 (raw float32 bytes)
num_scale_bytes = HEAD_DIM * 4 // QUANT_BLOCK_SIZE # 4
dst_scale = torch.zeros(
num_tokens, num_scale_bytes, dtype=torch.uint8, device=device
)
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
# ── Manual dequant ──────────────────────────────────────────────────
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
k_recovered = k_fp8 * scale # [num_tokens, 128]
# ── Compare ─────────────────────────────────────────────────────────
diff = (k_recovered - k.float()).abs()
k_abs = k.float().abs()
for t in range(num_tokens):
amax = k_abs[t].max().clamp(min=1e-4).item()
# UE8M0: scale = 2^ceil(log2(amax / 448))
exponent = math.ceil(math.log2(amax / 448.0))
ue8m0_scale = 2.0**exponent
# FP8 e4m3 (3-bit mantissa): worst-case error = 16 * scale
max_allowed = 16.0 * ue8m0_scale
token_diff = diff[t].max().item()
assert token_diff <= max_allowed, (
f"Token {t} diff {token_diff} exceeds max_allowed "
f"{max_allowed} (scale={ue8m0_scale})"
)
def test_indexer_gather_accepts_upper_bound_output():
"""Gather only exact cu_seq_lens even when dst is over-allocated."""
head_dim = 128
quant_block_size = 128
cache_stride = head_dim + head_dim * 4 // quant_block_size
valid_tokens = 9
upper_bound_tokens = 13
block_size = 16
num_blocks = 2
sentinel = 123
device = "cuda"
k = torch.randn(valid_tokens, head_dim, dtype=torch.bfloat16, device=device)
kv_cache = torch.zeros(
num_blocks, block_size, cache_stride, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(valid_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, quant_block_size, "ue8m0")
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
)
cu_seq_lens = torch.tensor([0, valid_tokens], dtype=torch.int32, device=device)
dst_k = torch.full(
(upper_bound_tokens, head_dim), sentinel, dtype=torch.uint8, device=device
)
num_scale_bytes = head_dim * 4 // quant_block_size
dst_scale = torch.full(
(upper_bound_tokens, num_scale_bytes),
sentinel,
dtype=torch.uint8,
device=device,
)
ops.cp_gather_indexer_k_quant_cache(
kv_cache, dst_k, dst_scale, block_table, cu_seq_lens
)
torch.accelerator.synchronize()
k_recovered = dst_k[:valid_tokens].view(torch.float8_e4m3fn).float() * dst_scale[
:valid_tokens
].view(torch.float32)
diff = (k_recovered - k.float()).abs()
max_allowed = (16.0 * dst_scale[:valid_tokens].view(torch.float32).max()).item()
assert diff.max().item() <= max_allowed
assert torch.all(dst_k[valid_tokens:] == sentinel)
assert torch.all(dst_scale[valid_tokens:] == sentinel)
# ── Test C: DeepseekV4 attention with values at different magnitudes ───────────
def test_deepseek_v4_quant_magnitude_range():
"""Test that quantization handles a range of magnitudes correctly."""
HEAD_DIM = 512
NOPE_DIM = 448
HEAD_BYTES = 584
block_size = 16
num_tokens = 4
num_blocks = 2
device = "cuda"
# Create inputs with varying magnitudes: small, medium, large
compressed_kv = torch.zeros(
num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
compressed_kv[0] = 0.001 # very small
compressed_kv[1] = 1.0 # unit scale
compressed_kv[2] = 100.0 # large
compressed_kv[3] = torch.randn(HEAD_DIM, dtype=torch.bfloat16, device=device)
k_cache = torch.zeros(
num_blocks, block_size, HEAD_BYTES, dtype=torch.uint8, device=device
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(
compressed_kv, k_cache.view(num_blocks, -1), slot_mapping, block_size
)
out = torch.zeros(1, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).unsqueeze(
0
)
dequantize_and_gather_k_cache(
out, k_cache, seq_lens, None, block_table, block_size, offset=0
)
recovered = out[0, :num_tokens]
# RoPE portion must be exact
rope_diff = (recovered[:, NOPE_DIM:] - compressed_kv[:, NOPE_DIM:]).abs().max()
assert rope_diff.item() == 0.0, f"RoPE diff {rope_diff.item()}"
# NoPE: relative error should be reasonable
for t in range(num_tokens):
orig = compressed_kv[t, :NOPE_DIM].float()
recv = recovered[t, :NOPE_DIM].float()
abs_diff = (recv - orig).abs().max().item()
magnitude = orig.abs().max().item()
if magnitude > 0.01:
rel_err = abs_diff / magnitude
assert rel_err < 0.15, (
f"Token {t}: rel_err={rel_err:.4f}, abs_diff={abs_diff:.6f}, "
f"magnitude={magnitude:.4f}"
)
@@ -0,0 +1,359 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel:
fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert
- Q side: per-head RMSNorm (no weight) + GPT-J RoPE on last 64 dims
- KV side: GPT-J RoPE on last 64 + UE8M0 FP8 quant + paged cache insert
We compare against:
- PyTorch reference for RMSNorm + GPT-J RoPE on Q
- Existing Triton `quantize_and_insert_k_cache` + round-trip via
`dequantize_and_gather_k_cache` for KV
The kernel is imported via
`torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert`.
"""
import pytest
import torch
from vllm.v1.attention.ops.deepseek_v4_ops import (
dequantize_and_gather_k_cache,
quantize_and_insert_k_cache,
)
# ── Constants matching the kernel ────────────────────────────────────────────
HEAD_DIM = 512
ROPE_DIM = 64
NOPE_DIM = HEAD_DIM - ROPE_DIM # 448
QUANT_BLOCK = 64
FP8_MAX = 448.0
HEAD_BYTES = NOPE_DIM + ROPE_DIM * 2 + 8 # 448 + 128 + 8 = 584
# ── PyTorch reference implementations ────────────────────────────────────────
def make_cos_sin_cache(max_pos: int, rope_dim: int, dtype, device):
"""Build a cos||sin cache matching DeepseekV4ScalingRotaryEmbedding layout.
cos_sin_cache[pos, :rope_dim/2] = cos(theta), [rope_dim/2:] = sin(theta).
"""
base = 10000.0
inv_freq = 1.0 / (
base
** (torch.arange(0, rope_dim, 2, dtype=torch.float32, device=device) / rope_dim)
)
t = torch.arange(max_pos, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j -> ij", t, inv_freq) # [max_pos, rope_dim/2]
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1) # [max_pos, rope_dim]
return cache.to(dtype)
def apply_rope_gptj_last_k(
x: torch.Tensor, positions: torch.Tensor, cos_sin_cache: torch.Tensor
) -> torch.Tensor:
"""GPT-J-style (interleaved-pair) RoPE on the LAST rope_dim elements.
x: [..., head_dim] float32
positions: [num_tokens] int64 (positions[i] corresponds to x[i, ...])
cos_sin_cache: [max_pos, rope_dim] float (cos|sin layout)
Returns rotated x (same shape/dtype).
"""
rope_dim = cos_sin_cache.shape[-1]
half = rope_dim // 2
head_dim = x.shape[-1]
nope_dim = head_dim - rope_dim
# Gather cos/sin for each token position: [num_tokens, rope_dim]
cs = cos_sin_cache[positions].to(torch.float32) # [N, rope_dim]
cos = cs[..., :half] # [N, half]
sin = cs[..., half:] # [N, half]
# Reshape leading dims so we can broadcast: x shape [..., head_dim].
# Bring token dim to front; assume x is [num_tokens, ..., head_dim].
# We rely on positions being per-token and all other dims sharing the same pos.
rope = x[..., nope_dim:].float() # [..., rope_dim]
# Make rope pairs: reshape last dim to [half, 2]
shape = rope.shape
rope = rope.reshape(*shape[:-1], half, 2)
even = rope[..., 0] # [..., half]
odd = rope[..., 1]
# Broadcast cos/sin over any heads dim in between. cos/sin are [N, half].
# Add singleton dims for intermediate axes.
for _ in range(rope.ndim - 3):
cos = cos.unsqueeze(1)
sin = sin.unsqueeze(1)
new_even = even * cos - odd * sin
new_odd = even * sin + odd * cos
rope_rotated = torch.stack((new_even, new_odd), dim=-1).reshape(shape)
out = x.clone().float()
out[..., nope_dim:] = rope_rotated
return out.to(x.dtype)
def rmsnorm_no_weight(x: torch.Tensor, eps: float) -> torch.Tensor:
"""RMSNorm with no learnable weight, matching
`RMSNorm(head_dim, has_weight=False)`."""
orig_dtype = x.dtype
xf = x.float()
variance = xf.pow(2).mean(dim=-1, keepdim=True)
return (xf * torch.rsqrt(variance + eps)).to(orig_dtype)
# ── Dispatch to the CUDA op (skip test cleanly if it isn't built in) ─────────
def _op_available() -> bool:
return hasattr(torch.ops._C, "fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or not _op_available(),
reason="CUDA not available or fused DeepseekV4 op not built in",
)
def _call_fused(q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs):
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
q, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs
)
# ── Test 1: Q path numerical parity ──────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("n_heads", [8, 64])
def test_q_path_matches_reference(num_tokens: int, n_heads: int):
torch.manual_seed(0)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
# Reference: RMSNorm (no weight) per head, then GPT-J RoPE on last 64.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
# Fused call with dummy KV tensors (KV branch will write slot_mapping=-1 → noop).
num_blocks = 2
bs = 16
kv = torch.zeros(num_tokens, HEAD_DIM, dtype=dtype, device=device)
k_cache = torch.zeros(
num_blocks, bs, HEAD_BYTES, dtype=torch.uint8, device=device
).view(num_blocks, -1)
slot_mapping = torch.full((num_tokens,), -1, dtype=torch.int64, device=device)
q_fused = q.clone()
_call_fused(q_fused, kv, k_cache, slot_mapping, positions, cos_sin_cache, eps, bs)
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
# ── Test 2: KV path round-trip byte/value parity ─────────────────────────────
def _ue8m0_per_block_scales(kv_roped_nope_f32: torch.Tensor, qblock: int):
"""Return per-token per-block max scale (used to bound FP8 error)."""
n_tok, nope = kv_roped_nope_f32.shape
n_blocks = nope // qblock
blocks = kv_roped_nope_f32.view(n_tok, n_blocks, qblock)
absmax = blocks.abs().amax(dim=-1).clamp(min=1e-4)
raw = absmax / FP8_MAX
exponent = torch.ceil(torch.log2(raw))
return torch.pow(2.0, exponent) # [n_tok, n_blocks]
@pytest.mark.parametrize("num_tokens", [1, 4, 17, 64])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_matches_reference(num_tokens: int, block_size: int):
torch.manual_seed(1)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# ── Reference path: RoPE on kv, then existing Triton quant+insert ──────
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# ── Fused path (dummy q, single head) ──────────────────────────────────
k_cache_fused = torch.zeros_like(k_cache_ref)
q_dummy = torch.zeros(num_tokens, 1, HEAD_DIM, dtype=dtype, device=device)
_call_fused(
q_dummy,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
# ── Round-trip compare via dequant+gather ──────────────────────────────
def _dequant(k_cache_2d):
num_reqs = 1
max_blocks = num_blocks
out = torch.zeros(
num_reqs, num_tokens, HEAD_DIM, dtype=torch.bfloat16, device=device
)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
block_table = torch.arange(
max_blocks, dtype=torch.int32, device=device
).unsqueeze(0)
# gather_lens arg is None (use seq_lens)
k_cache_3d = k_cache_2d.view(num_blocks, block_size, HEAD_BYTES)
dequantize_and_gather_k_cache(
out, k_cache_3d, seq_lens, None, block_table, block_size, offset=0
)
return out[0, :num_tokens]
recovered_ref = _dequant(k_cache_ref)
recovered_fused = _dequant(k_cache_fused)
# NoPE: per-block UE8M0 FP8 error bound (half-ULP at max = 16 * scale).
scales = _ue8m0_per_block_scales(kv_ref[:, :NOPE_DIM].float(), QUANT_BLOCK)
for t in range(num_tokens):
max_allowed = 16.0 * scales[t].max().item()
diff_ref = (
(recovered_ref[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
)
diff_fused = (
(recovered_fused[t, :NOPE_DIM] - kv_ref[t, :NOPE_DIM]).abs().max().item()
)
assert diff_ref <= max_allowed, (
f"ref NoPE token {t} diff {diff_ref} > {max_allowed}"
)
assert diff_fused <= max_allowed, (
f"fused NoPE token {t} diff {diff_fused} > {max_allowed}"
)
# RoPE region: bf16 stored exactly → zero diff.
rope_diff = (recovered_fused[:, NOPE_DIM:] - kv_ref[:, NOPE_DIM:]).abs().max()
assert rope_diff.item() == 0.0, f"RoPE portion not exact: {rope_diff.item()}"
# Exact byte equality of the two cache buffers — strong parity.
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
# ── Test 2b: DP padding (slot_mapping shorter than q/kv) ─────────────────────
@pytest.mark.parametrize("num_tokens", [4, 17])
@pytest.mark.parametrize("pad", [1, 5])
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
tokens in the KV branch while still running Q-norm+RoPE on all rows."""
torch.manual_seed(3)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
total = num_tokens + pad
kv = torch.randn(total, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(total, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# Reference: only the first num_tokens kv rows get inserted.
kv_ref = apply_rope_gptj_last_k(
kv[:num_tokens], positions[:num_tokens], cos_sin_cache
)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# Fused: pass full-sized q/kv/positions, shorter slot_mapping.
q_dummy = torch.zeros(total, 1, HEAD_DIM, dtype=dtype, device=device)
k_cache_fused = torch.zeros_like(k_cache_ref)
_call_fused(
q_dummy,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
# ── Test 3: combined single-call Q + KV parity ───────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 4, 17])
@pytest.mark.parametrize("n_heads", [8, 64])
@pytest.mark.parametrize("block_size", [16, 64])
def test_combined_q_and_kv(num_tokens: int, n_heads: int, block_size: int):
torch.manual_seed(2)
device = "cuda"
dtype = torch.bfloat16
eps = 1e-6
max_pos = 4096
q = torch.randn(num_tokens, n_heads, HEAD_DIM, dtype=dtype, device=device)
kv = torch.randn(num_tokens, HEAD_DIM, dtype=dtype, device=device)
positions = torch.arange(num_tokens, dtype=torch.int64, device=device)
cos_sin_cache = make_cos_sin_cache(max_pos, ROPE_DIM, torch.float32, device)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
# Reference.
q_ref = rmsnorm_no_weight(q, eps)
q_ref = apply_rope_gptj_last_k(q_ref, positions, cos_sin_cache)
kv_ref = apply_rope_gptj_last_k(kv, positions, cos_sin_cache)
k_cache_ref = torch.zeros(
num_blocks, block_size * HEAD_BYTES, dtype=torch.uint8, device=device
)
quantize_and_insert_k_cache(
kv_ref, k_cache_ref, slot_mapping, block_size=block_size
)
# Fused single call.
q_fused = q.clone()
k_cache_fused = torch.zeros_like(k_cache_ref)
_call_fused(
q_fused,
kv,
k_cache_fused,
slot_mapping,
positions,
cos_sin_cache,
eps,
block_size,
)
torch.testing.assert_close(q_fused, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_cache_fused, k_cache_ref, rtol=0, atol=0)
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for fused_indexer_q_rope_quant.
Compares the fused Triton kernel against the unfused reference flow used by
the DeepseekV4 indexer in model_tracking:
q_rot = ops.rotary_embedding(positions, q, None, head_dim, cos_sin_cache,
is_neox_style=False,
rope_dim_offset=head_dim - rope_dim)
q_fp8, q_scale = per_token_group_quant_fp8(q_rot, head_dim, use_ue8m0=True)
weights_out = weights * q_scale * softmax_scale * head_scale
Expects bit-exact equality on both q_fp8 and weights_out.
"""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
from vllm.v1.attention.ops.deepseek_v4_ops.fused_indexer_q import (
fused_indexer_q_rope_quant,
)
HEAD_DIM = 128
ROPE_DIM = 64
N_HEAD = 64
MAX_POS = 4096
def _reference(
positions: torch.Tensor,
q: torch.Tensor,
cos_sin_cache: torch.Tensor,
weights: torch.Tensor,
softmax_scale: float,
head_scale: float,
) -> tuple[torch.Tensor, torch.Tensor]:
q_rot = q.clone()
ops.rotary_embedding(
positions,
q_rot,
None,
HEAD_DIM,
cos_sin_cache,
False, # is_neox_style=False → GPT-J interleaved
HEAD_DIM - ROPE_DIM, # rope_dim_offset → rotate the tail
False,
)
q_fp8, q_scale = per_token_group_quant_fp8(
q_rot.view(-1, HEAD_DIM).contiguous(),
HEAD_DIM,
use_ue8m0=True,
)
q_fp8 = q_fp8.view(-1, N_HEAD, HEAD_DIM)
q_scale = q_scale.view(-1, N_HEAD)
weights_out = weights.to(torch.float32) * q_scale * softmax_scale * head_scale
return q_fp8, weights_out
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 257])
@pytest.mark.parametrize("cache_dtype", [torch.float32, torch.bfloat16])
@torch.inference_mode()
def test_fused_indexer_q_rope_quant_matches_unfused(num_tokens, cache_dtype):
device = "cuda"
torch.manual_seed(0)
q = torch.randn(num_tokens, N_HEAD, HEAD_DIM, dtype=torch.bfloat16, device=device)
positions = torch.randint(
0, MAX_POS, (num_tokens,), dtype=torch.int64, device=device
)
cos_sin_cache = torch.randn(MAX_POS, ROPE_DIM, dtype=cache_dtype, device=device)
weights = torch.randn(num_tokens, N_HEAD, dtype=torch.bfloat16, device=device)
softmax_scale = HEAD_DIM**-0.5
head_scale = N_HEAD**-0.5
q_fp8_ref, weights_ref = _reference(
positions, q, cos_sin_cache, weights, softmax_scale, head_scale
)
q_fp8_fused, weights_fused = fused_indexer_q_rope_quant(
positions, q.clone(), cos_sin_cache, weights, softmax_scale, head_scale
)
# fp8 tensors aren't directly comparable via torch.equal — reinterpret as int8.
ref_bits = q_fp8_ref.view(torch.int8)
fused_bits = q_fp8_fused.view(torch.int8)
assert torch.equal(ref_bits, fused_bits), (
f"q_fp8 mismatch: "
f"{(ref_bits != fused_bits).sum().item()} / {ref_bits.numel()} bytes differ"
)
assert torch.equal(weights_ref, weights_fused), (
f"weights mismatch: max abs diff "
f"{(weights_ref - weights_fused).abs().max().item()}"
)
@@ -0,0 +1,908 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the fused inverse RoPE + block-scaled FP8 quantization kernel.
Tests compare the fused kernel against a reference implementation built from
the existing separate operations (inverse RoPE via rotate_neox + FP8 quant
via per_token_group_quant_fp8).
The reference faithfully reproduces the exact flow in deepseek_v4_attention.py:295-310:
1. Apply inverse RoPE (NeoX style, last rope_dim=64 dims of each head)
2. Reshape [T, H, head_dim] -> [T, G, D]
3. Transpose+flatten to [G*T, D], quantize, reshape back
4. Return o_fp8 and o_scale with strides (D, T*D, 1) and (S, T*S, 1)
(non-contiguous [T, G, ...] view backed by contiguous [G, T, ...] memory)
Usage:
pytest tests/kernels/test_fused_inv_rope_fp8_quant.py -v
"""
import pytest
import torch
from vllm.v1.attention.ops.deepseek_v4_ops import fused_inv_rope_fp8_quant
# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
NOPE_DIM = 448
ROPE_DIM = 64
QUANT_GROUP_SIZE = 128
FP8_MAX = 448.0 # torch.finfo(torch.float8_e4m3fn).max
FP8_DTYPE = torch.float8_e4m3fn
EPS = 1e-10
# =========================================================================
# Helpers
# =========================================================================
def assert_dequant_close(
fp8_a: torch.Tensor,
scale_a: torch.Tensor,
fp8_b: torch.Tensor,
scale_b: torch.Tensor,
msg: str = "",
):
"""Compare two FP8-quantized tensors via their dequantized values.
Uses cosine-similarity-based diff (same as deep_gemm calc_diff).
Both fused and reference paths rotate in fp32 using an fp32
cos_sin_cache, so differences are only fp32 ordering ULPs that can
occasionally shift FP8 values at quantization boundaries.
"""
S = scale_a.shape[-1]
shape = fp8_a.shape
dq_a = fp8_a.float() * scale_a.unsqueeze(-1).expand(
*shape[:-1], S, QUANT_GROUP_SIZE
).reshape(shape)
dq_b = fp8_b.float() * scale_b.unsqueeze(-1).expand(
*shape[:-1], S, QUANT_GROUP_SIZE
).reshape(shape)
# Cosine diff: 1 - cos_sim (0 = identical, higher = worse)
dq_a_flat = dq_a.flatten().float()
dq_b_flat = dq_b.flatten().float()
cos_sim = torch.nn.functional.cosine_similarity(
dq_a_flat.unsqueeze(0), dq_b_flat.unsqueeze(0)
).item()
diff = 1.0 - cos_sim
assert diff < 1e-4, f"Dequant diff too large: {diff:.8f} (expected < 1e-4). {msg}"
def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
"""GPT-J style rotation: interleaved pairs, negate-swap.
Matches vllm/model_executor/layers/rotary_embedding/common.py:23-27.
DeepseekV4 uses is_neox_style=False, so this is the correct rotation.
"""
x1 = x[..., ::2]
x2 = x[..., 1::2]
x = torch.stack((-x2, x1), dim=-1)
return x.flatten(-2)
def make_cos_sin_cache(
max_pos: int,
rope_dim: int = ROPE_DIM,
dtype: torch.dtype = torch.float32,
device: str = "cuda",
) -> torch.Tensor:
"""Create a synthetic cos_sin_cache matching the layout used by
DeepseekV4ScalingRotaryEmbedding._compute_cos_sin_cache.
Shape: [max_pos, rope_dim] where first half is cos, second half is sin.
The fused kernel requires fp32; callers can override dtype if passing
the cache into the bf16-only paths.
"""
half = rope_dim // 2
# Use random but bounded frequencies so cos/sin are well-behaved
inv_freq = 1.0 / (
10000.0 ** (torch.arange(0, half, device=device, dtype=torch.float32) / half)
)
t = torch.arange(max_pos, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv_freq) # [max_pos, half]
cos = freqs.cos()
sin = freqs.sin()
cache = torch.cat((cos, sin), dim=-1) # [max_pos, rope_dim]
return cache.to(dtype)
def reference_inv_rope(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
) -> torch.Tensor:
"""Apply inverse RoPE to the last rope_dim dimensions of each head.
Matches the GPT-J inverse rotation in pos_encoding_kernels.cu, which
promotes the cache to fp32 and performs the rotation in fp32. The
result is cast back to the input dtype.
Args:
o: [T, H, head_dim] bf16
positions: [T] int64
cos_sin_cache: [max_pos, rope_dim] fp32
Returns:
o with inverse RoPE applied on the rope portion (bf16).
"""
assert cos_sin_cache.dtype == torch.float32
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
half = rope_dim // 2
cos = cos_sin[:, :half]
sin = cos_sin[:, half:]
# GPT-J style: repeat_interleave (not repeat) to match interleaved pairs
cos = cos.repeat_interleave(2, dim=-1).unsqueeze(1)
sin = sin.repeat_interleave(2, dim=-1).unsqueeze(1)
sin = -sin # inverse
o_pass = o[..., :nope_dim]
o_rot_f32 = o[..., nope_dim:].float()
o_rot_f32 = o_rot_f32 * cos + rotate_gptj(o_rot_f32) * sin
o_rot = o_rot_f32.to(o.dtype)
return torch.cat([o_pass, o_rot], dim=-1)
def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-block UE8M0 FP8 quantization in pure float32.
Matches the Triton kernel logic exactly:
absmax -> 2^ceil(log2(absmax / fp8_max)) -> clamp(x / scale) -> fp8
Args:
x_f32: [..., quant_group_size] float32 — one or more 128-element blocks.
Returns:
x_fp8: same shape, float8_e4m3fn
scales: [...] float32, one scale per block
"""
assert x_f32.shape[-1] == QUANT_GROUP_SIZE
absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS)
scale_raw = absmax * (1.0 / FP8_MAX)
scale = torch.exp2(torch.ceil(torch.log2(scale_raw)))
x_scaled = (x_f32 / scale).clamp(-FP8_MAX, FP8_MAX)
x_fp8 = x_scaled.to(FP8_DTYPE)
return x_fp8, scale.squeeze(-1)
def reference_inv_rope_fp8_quant(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
n_groups: int,
heads_per_group: int,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
quant_group_size: int = QUANT_GROUP_SIZE,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Full reference: inverse RoPE in fp32 + UE8M0 FP8 quant in fp32.
Mimics the Triton kernel's precision path exactly:
Load bf16 -> cast to fp32 -> apply inverse RoPE with fp32 cos/sin ->
UE8M0 quant in fp32 -> write fp8 + scale
Returns:
o_fp8: [T, G, D] FP8 with strides (D, T*D, 1)
o_scale: [T, G, S] FP32 with strides (S, T*S, 1)
"""
assert cos_sin_cache.dtype == torch.float32
T, _H, head_dim = o.shape
d = heads_per_group * head_dim
S = d // quant_group_size
half_rope = rope_dim // 2
chunks_per_head = head_dim // quant_group_size
# Reshape [T, H, head_dim] -> [T, G, heads_per_group, head_dim]
o_4d = o.view(T, n_groups, heads_per_group, head_dim)
# Lookup cos/sin directly in fp32
cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32
cos = cos_sin[:, :half_rope] # [T, half_rope] fp32
sin = cos_sin[:, half_rope:] # [T, half_rope] fp32
# Allocate outputs in [G, T, ...] contiguous layout
fp8_buf = torch.empty(n_groups, T, d, dtype=FP8_DTYPE, device=o.device)
scale_buf = torch.empty(n_groups, T, S, dtype=torch.float32, device=o.device)
# Process each quant block, matching the Triton kernel's per-program logic
for g in range(n_groups):
for qb in range(S):
head_in_group = qb // chunks_per_head
chunk_in_head = qb % chunks_per_head
offset = chunk_in_head * quant_group_size
# Load 128 bf16 elements and promote to fp32 for rotation+quant
block = o_4d[:, g, head_in_group, offset : offset + quant_group_size]
x = block.float()
# Apply inverse RoPE in fp32 if this is the last chunk
# GPT-J style: interleaved pairs (even=x, odd=y)
if chunk_in_head == chunks_per_head - 1:
rope_start = nope_dim % quant_group_size # 64
rope_region = x[:, rope_start:].clone()
x_vals = rope_region[:, ::2]
y_vals = rope_region[:, 1::2]
x_new = x_vals * cos + y_vals * sin
y_new = y_vals * cos - x_vals * sin
x = x.clone()
x[:, rope_start::2] = x_new
x[:, rope_start + 1 :: 2] = y_new
# UE8M0 quant in fp32
x_fp8, scale = _ref_ue8m0_quant_block(x)
# Write to [G, T, D] contiguous memory
fp8_buf[g, :, qb * quant_group_size : (qb + 1) * quant_group_size] = x_fp8
scale_buf[g, :, qb] = scale
# Return transposed views
return fp8_buf.transpose(0, 1), scale_buf.transpose(0, 1)
# =========================================================================
# Tests
# =========================================================================
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8), (32, 4), (128, 8)],
ids=["H64_G8", "H32_G4", "H128_G8"],
)
@pytest.mark.parametrize("seed", [0, 42])
@torch.inference_mode()
def test_correctness(num_tokens, num_heads, n_groups, seed):
"""Compare fused kernel against reference for FP8 values and scales."""
torch.manual_seed(seed)
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
# Create inputs
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(
max_pos, ROPE_DIM, dtype=torch.float32, device=device
)
# Reference
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Fused kernel
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Check shapes
d = heads_per_group * HEAD_DIM
S = d // QUANT_GROUP_SIZE
assert ref_fp8.shape == (num_tokens, n_groups, d)
assert fused_fp8.shape == (num_tokens, n_groups, d)
assert ref_scale.shape == (num_tokens, n_groups, S)
assert fused_scale.shape == (num_tokens, n_groups, S)
# Scales: exact match (both use identical UE8M0 algorithm)
# Scales may differ by one UE8M0 step (factor of 2) if fp32 rotation
# ordering shifts absmax across a power-of-2 boundary. Check ratio is
# close to 1.
scale_ratio = fused_scale / ref_scale.clamp(min=1e-30)
assert scale_ratio.max() <= 2.0 and scale_ratio.min() >= 0.5, (
f"Scale ratio out of [0.5, 2]: min={scale_ratio.min():.4f} "
f"max={scale_ratio.max():.4f}"
)
# Compare via dequant (Triton vs PyTorch fp32 may differ by ULPs)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8), (128, 8)],
ids=["H64_G8", "H128_G8"],
)
@torch.inference_mode()
def test_output_strides(num_tokens, num_heads, n_groups):
"""Verify fused output layout:
- FP8: logical [T, G, D] backed by contiguous [G, T, D].
- Scale: MN-major TMA-aligned (column-major: T-stride=1).
"""
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# FP8: logical [T, G, D] backed by [G, T, D] row-major
d = heads_per_group * HEAD_DIM
expected_fp8_stride = (d, num_tokens * d, 1)
assert fused_fp8.stride() == expected_fp8_stride, (
f"FP8 stride mismatch: got {fused_fp8.stride()}, expected {expected_fp8_stride}"
)
# Scale: MN-major TMA-aligned layout. After fp8_einsum permutes
# [T,G,S] -> [G,T,S], T-dim should have stride 1.
# Our output is [T,G,S] = transpose of [G,T,S].
# So fused_scale.permute(1,0,2) should have T-stride=1.
perm = fused_scale.permute(1, 0, 2) # [G, T, S]
assert perm.stride(1) == 1 or num_tokens == 1, (
f"Scale T-stride (after permute to [G,T,S]) should be 1, got {perm.stride(1)}"
)
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128])
@torch.inference_mode()
def test_per_group_contiguity(num_tokens):
"""FP8 per-group slices must be contiguous. Scale per-group slices
are column-major (T-stride=1) — not row-major contiguous, which is
correct for TMA loads."""
num_heads, n_groups = 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
for g in range(n_groups):
fp8_slice = fused_fp8[:, g, :]
assert fp8_slice.is_contiguous(), (
f"o_fp8[:, {g}, :] is not contiguous: "
f"shape={list(fp8_slice.shape)}, stride={list(fp8_slice.stride())}"
)
@torch.inference_mode()
def test_scales_are_power_of_two():
"""Verify all scales are exact powers of 2 (UE8M0 property)."""
num_tokens, num_heads, n_groups = 32, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
_, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# log2 of a power-of-two is an exact integer
log2_scales = torch.log2(fused_scale)
residual = (log2_scales - log2_scales.round()).abs()
assert residual.max() < 1e-5, (
f"Not all scales are powers of 2: max log2 residual = {residual.max().item()}"
)
@torch.inference_mode()
def test_nope_dims_unchanged():
"""Nope dimensions (first 448 per head) should only be quantized,
not rotated. Verify by dequantizing and comparing against
quantize-only reference (no RoPE)."""
num_tokens, num_heads, n_groups = 16, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# Fused kernel result
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Reference: quantize without RoPE (identity rotation)
# Create a zero-sin cache so RoPE is identity
zero_cache = torch.zeros_like(cos_sin_cache)
half = ROPE_DIM // 2
zero_cache[:, :half] = 1.0 # cos = 1
# sin = 0 (already zero)
norope_fp8, norope_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
zero_cache,
n_groups,
heads_per_group,
)
# Extract nope quant blocks only (first 3 of every 4 blocks per head)
chunks_per_head = HEAD_DIM // QUANT_GROUP_SIZE # 4
for h in range(heads_per_group):
for c in range(chunks_per_head - 1): # skip last chunk (has rope)
qb = h * chunks_per_head + c
start = qb * QUANT_GROUP_SIZE
end = start + QUANT_GROUP_SIZE
fused_nope = fused_fp8[:, :, start:end].view(torch.uint8)
norope_nope = norope_fp8[:, :, start:end].view(torch.uint8)
assert torch.equal(fused_nope, norope_nope), (
f"Nope block (head={h}, chunk={c}) differs between "
f"fused and no-rope reference"
)
fused_s = fused_scale[:, :, qb]
norope_s = norope_scale[:, :, qb]
assert torch.equal(fused_s, norope_s), (
f"Nope scale (head={h}, chunk={c}) differs"
)
@torch.inference_mode()
def test_single_token():
"""Edge case: single token."""
num_tokens, num_heads, n_groups = 1, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.tensor([42], device=device, dtype=torch.long)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_zero_positions():
"""Edge case: all positions are 0."""
num_tokens, num_heads, n_groups = 16, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.zeros(num_tokens, device=device, dtype=torch.long)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_large_values():
"""Edge case: values near FP8 saturation to test clamping."""
num_tokens, num_heads, n_groups = 8, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
# Create inputs with large values that will saturate FP8
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
o = o * 1000.0 # scale up to force saturation
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
ref_fp8, ref_scale = reference_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
assert_dequant_close(ref_fp8, ref_scale, fused_fp8, fused_scale)
@torch.inference_mode()
def test_dequant_numerical_accuracy():
"""Verify dequantized values are close to the original (after inv RoPE)."""
num_tokens, num_heads, n_groups = 32, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# Get the post-inv-RoPE values (ground truth before quantization)
o_after_rope = reference_inv_rope(o.clone(), positions, cos_sin_cache)
d = heads_per_group * HEAD_DIM
o_after_rope = o_after_rope.view(num_tokens, n_groups, d)
# Get fused quantized output
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Dequantize: broadcast scale [T, G, S] to [T, G, D] via repeat
S = d // QUANT_GROUP_SIZE
scale_expanded = (
fused_scale.unsqueeze(-1)
.expand(num_tokens, n_groups, S, QUANT_GROUP_SIZE)
.reshape(num_tokens, n_groups, d)
)
dequant = fused_fp8.float() * scale_expanded
# Check relative error.
# FP8 e4m3 with UE8M0 (power-of-two scales that round UP) quantizes more
# coarsely than optimal scaling. Both paths rotate in fp32, so the bulk
# of the error comes from UE8M0 quantization itself (~10-12% typical).
o_gt = o_after_rope.transpose(0, 1).contiguous().transpose(0, 1)
dequant_contig = dequant.transpose(0, 1).contiguous().transpose(0, 1)
abs_err = (dequant_contig.float() - o_gt.float()).abs()
rel_err = abs_err / (o_gt.float().abs().clamp(min=1e-6))
mean_rel_err = rel_err.mean().item()
assert mean_rel_err < 0.15, (
f"Mean relative error too high: {mean_rel_err:.4f} (expected < 0.15)"
)
def _unfused_inv_rope_fp8_quant(
o: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
n_groups: int,
heads_per_group: int,
nope_dim: int = NOPE_DIM,
rope_dim: int = ROPE_DIM,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Unfused path matching deepseek_v4_attention.py:295-310.
Uses the production CUDA RoPE kernel + per_token_group_quant_fp8.
"""
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
head_dim = o.shape[-1]
rope_dim_offset = head_dim - rope_dim
# Step 1: In-place CUDA RoPE (same as production)
ops.rotary_embedding(
positions,
o,
None,
head_dim,
cos_sin_cache,
False, # is_neox=False for DeepseekV4 (GPT-J style)
rope_dim_offset=rope_dim_offset,
inverse=True,
)
# Step 2: Reshape + quant + reshape (same as production)
T = o.shape[0]
d = heads_per_group * head_dim
o = o.view(T, n_groups, -1)
o_flat = o.transpose(0, 1).contiguous().reshape(-1, d)
o_fp8, o_scale = per_token_group_quant_fp8(
o_flat,
group_size=QUANT_GROUP_SIZE,
use_ue8m0=True,
)
o_fp8 = o_fp8.view(n_groups, T, d).transpose(0, 1)
o_scale = o_scale.view(n_groups, T, -1).transpose(0, 1)
return o_fp8, o_scale
# =========================================================================
# End-to-end test including fp8_einsum
# =========================================================================
@pytest.mark.parametrize("num_tokens", [1, 7, 32, 128, 1024])
@pytest.mark.parametrize(
"num_heads,n_groups",
[(64, 8)],
ids=["H64_G8"],
)
@torch.inference_mode()
def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
"""End-to-end: fused inv_rope+quant → fp8_einsum must match
unfused CUDA_rope+quant → fp8_einsum bitwise.
This catches stride/layout bugs that only manifest when the einsum
kernel actually consumes the quantized activations.
"""
from deep_gemm.utils.math import ceil_div
from vllm.utils.deep_gemm import (
fp8_einsum,
per_block_cast_to_fp8,
transform_sf_into_required_layout,
)
heads_per_group = num_heads // n_groups
d = heads_per_group * HEAD_DIM
o_lora_rank = 1024
max_pos = 4096
device = "cuda"
torch.manual_seed(0)
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(
0, max_pos, (num_tokens,), device=device, dtype=torch.long
)
cos_sin_cache = make_cos_sin_cache(max_pos, device=device)
# -- Weight quantization (shared between both paths) --
w = torch.randn(n_groups, o_lora_rank, d, device=device, dtype=torch.bfloat16)
w_fp8 = torch.empty_like(w, dtype=torch.float8_e4m3fn)
w_scale = torch.empty(
n_groups,
ceil_div(o_lora_rank, 128),
ceil_div(d, 128),
device=device,
dtype=torch.float32,
)
for g in range(n_groups):
w_fp8[g], w_scale[g] = per_block_cast_to_fp8(w[g], use_ue8m0=True)
recipe = (1, 1, 128)
w_scale_t = transform_sf_into_required_layout(
sf=w_scale,
mn=o_lora_rank,
k=d,
recipe=(1, 128, 128),
num_groups=n_groups,
is_sfa=False,
)
# -- UNFUSED path --
ref_fp8, ref_scale = _unfused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
z_ref = torch.empty(
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
)
fp8_einsum(
"bhr,hdr->bhd", (ref_fp8, ref_scale), (w_fp8, w_scale_t), z_ref, recipe=recipe
)
# -- FUSED path --
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
z_fused = torch.empty(
num_tokens, n_groups, o_lora_rank, device=device, dtype=torch.bfloat16
)
fp8_einsum(
"bhr,hdr->bhd",
(fused_fp8, fused_scale),
(w_fp8, w_scale_t),
z_fused,
recipe=recipe,
)
# -- Checks --
# Einsum output: Triton and CUDA both rotate in fp32 now, so diffs
# come from fp32 ordering and UE8M0 boundary shifts only.
# Use relative diff (same metric as test_fp8_einsum.py).
from deep_gemm.testing import calc_diff
z_diff = calc_diff(z_fused, z_ref)
assert z_diff < 0.01, (
f"Einsum output diff too large: {z_diff:.6f} (expected < 0.01)"
)
@pytest.mark.parametrize("num_tokens", [1, 32, 256])
@torch.inference_mode()
def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config):
"""Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style,
mscale=0, YaRN scaling) matching the production config."""
num_heads = 64
n_groups = 8
heads_per_group = num_heads // n_groups
device = "cuda"
torch.manual_seed(0)
# Build YaRN-scaled cos_sin_cache matching real DeepSeek V3/V4 config
# (mscale=0 → mscale=1.0, so no magnitude scaling)
from vllm.model_executor.layers.rotary_embedding.common import (
yarn_find_correction_range,
yarn_linear_ramp_mask,
)
scaling_factor = 16
base = 10000.0
max_pos = 65536
beta_fast, beta_slow = 32, 1
pos_freqs = base ** (
torch.arange(0, ROPE_DIM, 2, dtype=torch.float32, device=device) / ROPE_DIM
)
inv_freq_extra = 1.0 / pos_freqs
inv_freq_interp = 1.0 / (scaling_factor * pos_freqs)
low, high = yarn_find_correction_range(
beta_fast, beta_slow, ROPE_DIM, base, max_pos
)
mask = 1 - yarn_linear_ramp_mask(low, high, ROPE_DIM // 2, dtype=torch.float32).to(
device
)
inv_freq = inv_freq_interp * (1 - mask) + inv_freq_extra * mask
t = torch.arange(max_pos * scaling_factor, device=device, dtype=torch.float32)
freqs = torch.outer(t, inv_freq)
# mscale=0 → yarn_get_mscale returns 1.0
cos_sin_cache = torch.cat([freqs.cos(), freqs.sin()], dim=-1) # fp32
o = torch.randn(
num_tokens, num_heads, HEAD_DIM, device=device, dtype=torch.bfloat16
)
positions = torch.randint(0, 4096, (num_tokens,), device=device, dtype=torch.long)
# UNFUSED: CUDA RoPE with is_neox=False (GPT-J)
from vllm import _custom_ops as ops
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
per_token_group_quant_fp8,
)
o_unfused = o.clone()
ops.rotary_embedding(
positions,
o_unfused,
None,
HEAD_DIM,
cos_sin_cache,
False, # is_neox=False (GPT-J style)
rope_dim_offset=NOPE_DIM,
inverse=True,
)
d = heads_per_group * HEAD_DIM
T = num_tokens
o_unfused = o_unfused.view(T, n_groups, d)
o_flat = o_unfused.transpose(0, 1).contiguous().reshape(-1, d)
ref_fp8, ref_scale = per_token_group_quant_fp8(
o_flat,
group_size=QUANT_GROUP_SIZE,
use_ue8m0=True,
)
ref_fp8 = ref_fp8.view(n_groups, T, d).transpose(0, 1)
ref_scale = ref_scale.view(n_groups, T, -1).transpose(0, 1)
# FUSED: use the real YaRN-scaled cos_sin_cache
fused_fp8, fused_scale = fused_inv_rope_fp8_quant(
o.clone(),
positions,
cos_sin_cache,
n_groups,
heads_per_group,
)
# Scales must match exactly (same UE8M0 algorithm)
# Compare via dequant (Triton bf16 rotation may differ from CUDA by 1 ULP)
assert_dequant_close(
ref_fp8, ref_scale, fused_fp8, fused_scale, msg="Real DeepSeek V4 rope"
)
+5 -10
View File
@@ -718,7 +718,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [2000, 6000, 30000, 80000],
"top_k": 2048,
"data_type": "random",
},
id="mixed_all_paths",
@@ -727,7 +726,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [2048, 4096, 8192, 16000],
"top_k": 2048,
"data_type": "random",
},
id="all_decode_medium",
@@ -736,7 +734,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [70000, 100000, 163840],
"top_k": 2048,
"data_type": "random",
},
id="all_large",
@@ -745,7 +742,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [32767, 32768, 32769, 32772],
"top_k": 2048,
"data_type": "random",
},
id="large_threshold_boundary",
@@ -754,7 +750,6 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [5000],
"top_k": 2048,
"data_type": "random",
},
id="single_row_medium",
@@ -772,15 +767,15 @@ def test_persistent_topk_stress() -> None:
pytest.param(
{
"seq_lens": [100, 2048, 10000, 80000],
"top_k": 2048,
"data_type": "random",
},
id="trivial_medium_large_mix",
),
],
)
@pytest.mark.parametrize("top_k", [512, 2048])
@torch.inference_mode()
def test_persistent_topk(test_config: dict) -> None:
def test_persistent_topk(test_config: dict, top_k: int) -> None:
"""
Tests specific to the persistent_topk kernel:
- Mixed medium/large rows in the same batch (dynamic per-row dispatch)
@@ -790,14 +785,15 @@ def test_persistent_topk(test_config: dict) -> None:
run_large_context_topk_test(
batch_size=len(test_config["seq_lens"]),
seq_lens=test_config["seq_lens"],
top_k=test_config["top_k"],
top_k=top_k,
data_type=test_config.get("data_type", "random"),
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@pytest.mark.parametrize("top_k", [512, 2048])
@torch.inference_mode()
def test_persistent_topk_padded_stride() -> None:
def test_persistent_topk_padded_stride(top_k: int) -> None:
"""
Test persistent_topk with padded logits (large stride, small seq_len)
to simulate the e2e CUDAGraph scenario where fp8_paged_mqa_logits
@@ -806,7 +802,6 @@ def test_persistent_topk_padded_stride() -> None:
set_random_seed(42)
torch.set_default_device("cuda:0")
top_k = 2048
batch_size = 4
padded_stride = 163840 # DeepSeek-V3.2 max_model_len
actual_seq_lens = [3000, 5000, 8000, 12000]
@@ -41,7 +41,9 @@ class DummyRouter(BaseRouter):
def routing_method_type(self) -> RoutingMethodType:
return RoutingMethodType.FUSED_TOPK
def _compute_routing(self, hidden_states, router_logits, indices_type):
def _compute_routing(
self, hidden_states, router_logits, indices_type, *, input_ids=None
):
topk_ids = torch.tensor([[1, 2], [3, 4]], dtype=torch.int64)
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
return topk_weights, topk_ids
+9
View File
@@ -260,6 +260,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
trust_remote_code=True,
),
"DeepseekV32ForCausalLM": _HfExamplesInfo("deepseek-ai/DeepSeek-V3.2-Exp"),
"DeepseekV4ForCausalLM": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash", is_available_online=False
),
"Ernie4_5ForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-0.3B-PT"),
"Ernie4_5_MoeForCausalLM": _HfExamplesInfo("baidu/ERNIE-4.5-21B-A3B-PT"),
"ExaoneForCausalLM": _HfExamplesInfo(
@@ -1482,6 +1485,12 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="luccafong/deepseek_mtp_draft_random",
trust_remote_code=True,
),
"DeepSeekV4MTPModel": _HfExamplesInfo(
"deepseek-ai/DeepSeek-V4-Flash",
speculative_model="deepseek-ai/DeepSeek-V4-Flash",
trust_remote_code=True,
is_available_online=False,
),
"ErnieMTPModel": _HfExamplesInfo(
"baidu/ERNIE-4.5-21B-A3B-PT",
trust_remote_code=True,
+184
View File
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.models.deepseek_v4 import (
DeepseekV4MegaMoEExperts,
_stage_deepseek_v4_mega_moe_inputs,
make_deepseek_v4_expert_params_mapping,
)
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="DeepSeek V4 MegaMoE requires CUDA",
)
def test_deepseek_v4_mega_moe_expert_mapping():
mapping = make_deepseek_v4_expert_params_mapping(2)
assert mapping == [
("experts.w13_", "experts.0.w1.", 0, "w1"),
("experts.w2_", "experts.0.w2.", 0, "w2"),
("experts.w13_", "experts.0.w3.", 0, "w3"),
("experts.w13_", "experts.1.w1.", 1, "w1"),
("experts.w2_", "experts.1.w2.", 1, "w2"),
("experts.w13_", "experts.1.w3.", 1, "w3"),
]
def test_deepseek_v4_mega_moe_ue8m0_uint8_to_float():
raw = torch.tensor([0, 126, 127, 128], dtype=torch.uint8)
decoded = DeepseekV4MegaMoEExperts._ue8m0_uint8_to_float(raw)
assert torch.equal(decoded.view(torch.int32), raw.to(torch.int32) << 23)
assert decoded[0].item() == 0.0
assert decoded[1].item() == 0.5
assert decoded[2].item() == 1.0
assert decoded[3].item() == 2.0
def test_deepseek_v4_mega_moe_weight_loader_uses_ep_expert_ownership():
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=4)
)
experts = DeepseekV4MegaMoEExperts(
vllm_config,
num_experts=4,
num_local_experts=2,
experts_start_idx=2,
top_k=2,
hidden_size=128,
intermediate_size=128,
)
nonlocal_weight = torch.ones(128, 64, dtype=torch.uint8)
assert (
experts.weight_loader(
experts.w13_weight,
nonlocal_weight,
"experts.w13_weight",
shard_id="w1",
expert_id=1,
return_success=True,
)
is False
)
w1 = torch.full((128, 64), 3, dtype=torch.uint8)
w3 = torch.full((128, 64), 7, dtype=torch.uint8)
w2 = torch.full((128, 64), 11, dtype=torch.uint8)
assert experts.weight_loader(
experts.w13_weight,
w1,
"experts.w13_weight",
shard_id="w1",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w13_weight,
w3,
"experts.w13_weight",
shard_id="w3",
expert_id=2,
return_success=True,
)
assert experts.weight_loader(
experts.w2_weight,
w2,
"experts.w2_weight",
shard_id="w2",
expert_id=2,
return_success=True,
)
assert torch.equal(experts.w13_weight[0, :128], w1)
assert torch.equal(experts.w13_weight[0, 128:], w3)
assert torch.equal(experts.w2_weight[0], w2)
assert torch.count_nonzero(experts.w13_weight[1]) == 0
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="DeepSeek V4 MegaMoE fused input staging requires CUDA.",
)
def test_deepseek_v4_mega_moe_fused_input_staging_is_bitwise_exact():
from vllm.third_party.deep_gemm.utils import per_token_cast_to_fp8
device = torch.device("cuda")
num_tokens = 7
hidden_size = 256
top_k = 8
generator = torch.Generator(device=device)
generator.manual_seed(0)
hidden_states = (
torch.randn(
num_tokens,
hidden_size,
device=device,
dtype=torch.float32,
generator=generator,
)
* 17.0
).to(torch.bfloat16)
hidden_states[0, :32] = 0
hidden_states[1, 32:64] = 1.0e-6
hidden_states[2, 64:96] = -1.0e-6
topk_ids = torch.randint(
0,
256,
(num_tokens, top_k),
device=device,
dtype=torch.int32,
generator=generator,
)
topk_weights = torch.randn(
num_tokens,
top_k,
device=device,
dtype=torch.float32,
generator=generator,
)
ref_x, ref_x_sf = per_token_cast_to_fp8(
hidden_states,
use_ue8m0=True,
gran_k=32,
use_packed_ue8m0=True,
)
ref_topk_idx = topk_ids.to(torch.int64)
ref_topk_weights = topk_weights.clone()
fused_x = torch.empty_like(ref_x)
fused_x_sf = torch.empty_like(ref_x_sf)
fused_topk_idx = torch.empty_like(ref_topk_idx)
fused_topk_weights = torch.empty_like(ref_topk_weights)
_stage_deepseek_v4_mega_moe_inputs(
hidden_states,
topk_weights,
topk_ids,
fused_x,
fused_x_sf,
fused_topk_idx,
fused_topk_weights,
)
torch.accelerator.synchronize()
assert torch.equal(fused_x.view(torch.uint8), ref_x.view(torch.uint8))
assert torch.equal(fused_x_sf, ref_x_sf)
assert torch.equal(fused_topk_idx, ref_topk_idx)
assert torch.equal(
fused_topk_weights.view(torch.uint8),
ref_topk_weights.view(torch.uint8),
)
@@ -6,6 +6,7 @@ from transformers import AutoTokenizer
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.reasoning import ReasoningParserManager
from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser
from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser
from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser
@@ -33,6 +34,12 @@ def test_parser_selection(tokenizer, thinking, expected_parser_type):
assert isinstance(parser._parser, expected_parser_type)
def test_deepseek_v4_reasoning_parser_alias():
parser_cls = ReasoningParserManager.get_reasoning_parser("deepseek_v4")
assert parser_cls is DeepSeekV3ReasoningParser
def test_identity_reasoning_parser_basic(tokenizer):
parser = IdentityReasoningParser(tokenizer)
@@ -0,0 +1,81 @@
{
"tools": [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a specific location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city name"
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "Temperature unit"
}
},
"required": ["location"]
}
}
},
{
"type": "function",
"function": {
"name": "search",
"description": "Search the web for information",
"parameters": {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
},
"num_results": {
"type": "integer",
"description": "Number of results to return"
}
},
"required": ["query"]
}
}
}
],
"messages": [
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "What's the weather in Beijing?"
},
{
"role": "assistant",
"reasoning": "The user wants to know the weather in Beijing. I should use the get_weather tool.",
"tool_calls": [
{
"id": "call_001",
"type": "function",
"function": {
"name": "get_weather",
"arguments": "{\"location\": \"Beijing\", \"unit\": \"celsius\"}"
}
}
]
},
{
"role": "tool",
"tool_call_id": "call_001",
"content": "{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"
},
{
"role": "assistant",
"reasoning": "Got the weather data. Let me format a nice response.",
"content": "The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."
}
]
}
@@ -0,0 +1,24 @@
[
{
"role": "system",
"content": "You are a helpful assistant."
},
{
"role": "user",
"content": "Hello"
},
{
"role": "assistant",
"reasoning": "The user said hello, I should greet back.",
"content": "Hi there! How can I help you?"
},
{
"role": "user",
"content": "What is the capital of France?"
},
{
"role": "assistant",
"reasoning": "The user asks about the capital of France. It is Paris.",
"content": "The capital of France is Paris."
}
]
@@ -0,0 +1,159 @@
[
{
"role": "system",
"content": "该助手为DeepSeek,由深度求索公司创造。"
},
{
"role": "latest_reminder",
"content": "2026-02-21,星期六,广州,App,中文"
},
{
"role": "developer",
"content": "小柴胡冲剂和布洛芬能一起吃吗?\n\nCITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】",
"tools": [
{
"type": "function",
"function": {
"name": "search",
"description": "Web search. Split multiple queries with '||'.",
"parameters": {
"type": "object",
"properties": {
"queries": {
"type": "string",
"description": "query1||query2"
}
},
"required": [
"queries"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
{
"type": "function",
"function": {
"name": "open",
"description": "Batch open IDs (format 【{id}†...】) or URLs.",
"parameters": {
"type": "object",
"properties": {
"open_list": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"description": "ID or URL",
"anyOf": [
{
"type": "integer"
},
{
"type": "string"
}
],
"default": -1
},
"cursor": {
"type": "integer",
"description": "",
"default": -1
},
"loc": {
"type": "integer",
"description": "Start line",
"default": -1
},
"num_lines": {
"type": "integer",
"description": "",
"default": -1
},
"view_source": {
"type": "boolean",
"description": "",
"default": false
}
},
"additionalProperties": false
},
"description": ""
}
},
"required": [
"open_list"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
},
{
"type": "function",
"function": {
"name": "find",
"description": "Find exact text pattern in pages.",
"parameters": {
"type": "object",
"properties": {
"find_list": {
"type": "array",
"items": {
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": ""
},
"cursor": {
"type": "integer",
"description": "",
"default": -1
}
},
"required": [
"pattern"
],
"additionalProperties": false
},
"description": ""
}
},
"required": [
"find_list"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#"
}
}
}
]
},
{
"role": "assistant",
"content": "",
"reasoning": "用户想知道小柴胡冲剂和布洛芬能否一起服用。",
"tool_calls": [
{
"type": "function",
"function": {
"name": "search",
"arguments": "{\"queries\": \"小柴胡冲剂 布洛芬 相互作用 一起吃\"}"
}
}
]
},
{
"role": "tool",
"content": "[0]"
},
{
"role": "assistant",
"content": "请及时就医。",
"reasoning": "现在开始组织回答。",
"tool_calls": []
}
]
@@ -0,0 +1,28 @@
[
{
"role": "system",
"content": "该助手为DeepSeek-V3,由深度求索公司创造。\n今天是2025年10月17日,星期五。"
},
{
"role": "latest_reminder",
"content": "2024-11-15,上海市,App,中文"
},
{
"role": "user",
"content": "热海大滚锅是世界著名温泉吗"
},
{
"role": "assistant",
"content": "关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。\n\n为了让你快速了解,我整理了一个简要的表格:\n\n| 维度 | 热海大滚锅的地位与特点 |\n| :--- | :--- |\n| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |\n| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |\n| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |\n| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |\n\n### 💡 游玩攻略与温馨提示\n\n如果你计划前往热海大滚锅,这里有一些实用信息供你参考:\n\n- **门票与开放时间**:\n - **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。\n - **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。\n\n- **特色体验**:\n - **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。\n - **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。\n\n- **注意事项**:\n - **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。\n - **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。\n\n希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。",
"mask": 1
},
{
"role": "user",
"content": "世界著名温泉有哪些",
"task": "action"
},
{
"role": "assistant",
"content": "Search"
}
]
@@ -0,0 +1,36 @@
<|begin▁of▁sentence|>
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
<|DSML|tool_calls>
<|DSML|invoke name="$TOOL_NAME">
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
...
</|DSML|invoke>
<|DSML|invoke name="$TOOL_NAME2">
...
</|DSML|invoke>
</|DSML|tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}}
{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
You are a helpful assistant.<|User|>What's the weather in Beijing?<|Assistant|><think>The user wants to know the weather in Beijing. I should use the get_weather tool.</think>
<|DSML|tool_calls>
<|DSML|invoke name="get_weather">
<|DSML|parameter name="location" string="true">Beijing</|DSML|parameter>
<|DSML|parameter name="unit" string="true">celsius</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>{"temperature": 22, "condition": "sunny", "humidity": 45}</tool_result><|Assistant|><think>Got the weather data. Let me format a nice response.</think>The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<|end▁of▁sentence|>
@@ -0,0 +1 @@
<|begin▁of▁sentence|>You are a helpful assistant.<|User|>Hello<|Assistant|></think>Hi there! How can I help you?<|end▁of▁sentence|><|User|>What is the capital of France?<|Assistant|><think>The user asks about the capital of France. It is Paris.</think>The capital of France is Paris.<|end▁of▁sentence|>
@@ -0,0 +1,38 @@
<|begin▁of▁sentence|>该助手为DeepSeek,由深度求索公司创造。<|latest_reminder|>2026-02-21,星期六,广州,App,中文<|User|>小柴胡冲剂和布洛芬能一起吃吗?
CITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】
## Tools
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<|DSML|tool_calls>" block like the following:
<|DSML|tool_calls>
<|DSML|invoke name="$TOOL_NAME">
<|DSML|parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</|DSML|parameter>
...
</|DSML|invoke>
<|DSML|invoke name="$TOOL_NAME2">
...
</|DSML|invoke>
</|DSML|tool_calls>
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
Otherwise, output directly after </think> with tool calls or final response.
### Available Tool Schemas
{"name": "search", "description": "Web search. Split multiple queries with '||'.", "parameters": {"type": "object", "properties": {"queries": {"type": "string", "description": "query1||query2"}}, "required": ["queries"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
{"name": "open", "description": "Batch open IDs (format 【{id}†...】) or URLs.", "parameters": {"type": "object", "properties": {"open_list": {"type": "array", "items": {"type": "object", "properties": {"id": {"description": "ID or URL", "anyOf": [{"type": "integer"}, {"type": "string"}], "default": -1}, "cursor": {"type": "integer", "description": "", "default": -1}, "loc": {"type": "integer", "description": "Start line", "default": -1}, "num_lines": {"type": "integer", "description": "", "default": -1}, "view_source": {"type": "boolean", "description": "", "default": false}}, "additionalProperties": false}, "description": ""}}, "required": ["open_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
{"name": "find", "description": "Find exact text pattern in pages.", "parameters": {"type": "object", "properties": {"find_list": {"type": "array", "items": {"type": "object", "properties": {"pattern": {"type": "string", "description": ""}, "cursor": {"type": "integer", "description": "", "default": -1}}, "required": ["pattern"], "additionalProperties": false}, "description": ""}}, "required": ["find_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}}
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
<|Assistant|><think>用户想知道小柴胡冲剂和布洛芬能否一起服用。</think>
<|DSML|tool_calls>
<|DSML|invoke name="search">
<|DSML|parameter name="queries" string="true">小柴胡冲剂 布洛芬 相互作用 一起吃</|DSML|parameter>
</|DSML|invoke>
</|DSML|tool_calls><|end▁of▁sentence|><|User|><tool_result>[0]</tool_result><|Assistant|><think>现在开始组织回答。</think>请及时就医。<|end▁of▁sentence|>
@@ -0,0 +1,29 @@
<|begin▁of▁sentence|>该助手为DeepSeek-V3,由深度求索公司创造。
今天是2025年10月17日,星期五。<|latest_reminder|>2024-11-15,上海市,App,中文<|User|>热海大滚锅是世界著名温泉吗<|Assistant|></think>关于热海大滚锅是否是世界著名温泉,可以这样总结:它在**中国乃至全球的地热奇观中占有重要地位**,但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**,而非普遍的温泉疗养体验。
为了让你快速了解,我整理了一个简要的表格:
| 维度 | 热海大滚锅的地位与特点 |
| :--- | :--- |
| **地质奇观** | **世界罕见**的高温地热系统,泉眼多、水温高、形态多样。 |
| **国内声誉** | **中国三大地热区之一**,**国家5A级旅游景区**,自古闻名(徐霞客曾游历并记载)。 |
| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一,但此说法流传不广,其国际声誉更多建立在地质独特性上。 |
| **核心体验** | **观赏地热奇观**(如97℃沸腾的“大滚锅”)、**体验温泉煮鸡蛋**。 |
### 💡 游玩攻略与温馨提示
如果你计划前往热海大滚锅,这里有一些实用信息供你参考:
- **门票与开放时间**:
- **门票**:景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐,价格会更高,例如约**288元**。
- **开放时间**:景区一般**08:00-18:00**开放,但具体时间可能变动,建议提前核实。
- **特色体验**:
- **温泉煮鸡蛋**:这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋(约5-8元/串),然后到“大滚锅”旁的指定区域蒸煮,几分钟便可熟食,趣味十足。
- **金汤足浴**:可以直接用从“大滚锅”流出的温泉水泡脚,缓解旅途疲劳。
- **注意事项**:
- **安全第一**:“大滚锅”水温极高,务必遵守游览规则,在指定区域内观赏,切勿随意触碰泉水。
- **规划行程**:建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路,出入口有观光车接送。
希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问,我很乐意提供进一步的信息。<|end▁of▁sentence|><|User|>世界著名温泉有哪些<|Assistant|></think><|action|>Search<|end▁of▁sentence|>
+224
View File
@@ -0,0 +1,224 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from pathlib import Path
from types import SimpleNamespace
import pytest
from vllm.entrypoints.chat_utils import parse_chat_messages
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.deepseek_v4 import get_deepseek_v4_tokenizer
from vllm.tokenizers.registry import TokenizerRegistry
FIXTURES_DIR = Path(__file__).parent / "fixtures" / "deepseek_v4"
class FakeHfTokenizer:
vocab_size = 100
def get_added_vocab(self) -> dict[str, int]:
return {"</think>": 100}
def encode(
self,
text: str,
add_special_tokens: bool = False,
**kwargs,
) -> list[int]:
self.last_encode = (text, add_special_tokens, kwargs)
return [len(text)]
def _tokenizer():
return get_deepseek_v4_tokenizer(FakeHfTokenizer())
def _model_config():
return SimpleNamespace(
multimodal_config=None,
allowed_local_media_path="",
allowed_media_domains=None,
)
def _load_reference_case(case_id: int):
data = json.loads((FIXTURES_DIR / f"test_input_{case_id}.json").read_text())
if isinstance(data, dict):
return data["messages"], data.get("tools")
return data, None
def _render_reference_case(case_id: int, **kwargs):
messages, tools = _load_reference_case(case_id)
conversation, _, _ = parse_chat_messages(
messages,
_model_config(),
content_format="string",
)
return _tokenizer().apply_chat_template(
conversation=conversation,
messages=messages,
tools=tools,
tokenize=False,
**kwargs,
)
def test_deepseek_v4_tokenizer_registered():
assert TokenizerRegistry.load_tokenizer_cls("deepseek_v4").__name__ == (
"DeepseekV4Tokenizer"
)
assert RENDERER_REGISTRY.load_renderer_cls("deepseek_v4").__name__ == (
"DeepseekV4Renderer"
)
def test_deepseek_v4_defaults_to_chat_mode():
prompt = _tokenizer().apply_chat_template(
[{"role": "user", "content": "Hello"}],
tokenize=False,
)
assert prompt == ("<|begin▁of▁sentence|><|User|>Hello<|Assistant|></think>")
@pytest.mark.parametrize("kwargs", [{"thinking": True}, {"enable_thinking": True}])
def test_deepseek_v4_enables_thinking_with_compatible_kwargs(kwargs):
prompt = _tokenizer().apply_chat_template(
[{"role": "user", "content": "Hello"}],
tokenize=False,
**kwargs,
)
assert prompt == ("<|begin▁of▁sentence|><|User|>Hello<|Assistant|><think>")
def test_deepseek_v4_uses_v4_tool_prompt_from_request_tools():
tools = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
prompt = _tokenizer().apply_chat_template(
[{"role": "user", "content": "Weather?"}],
tools=tools,
tokenize=False,
)
assert "## Tools" in prompt
assert "<|DSML|tool_calls>" in prompt
assert "</|DSML|tool_calls>" in prompt
assert "function_calls" not in prompt
assert '"name": "get_weather"' in prompt
assert prompt.endswith("<|User|>Weather?<|Assistant|></think>")
def test_deepseek_v4_renders_parsed_history_tool_arguments():
messages = [
{"role": "user", "content": "List the repo"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {
"name": "str_replace_editor",
"arguments": '{"command": "view", "path": "/testbed"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": "file list",
},
]
tools = [
{
"type": "function",
"function": {
"name": "str_replace_editor",
"description": "Edit files",
"parameters": {
"type": "object",
"properties": {
"command": {"type": "string"},
"path": {"type": "string"},
},
"required": ["command", "path"],
},
},
}
]
conversation, _, _ = parse_chat_messages(
messages,
_model_config(),
content_format="string",
)
prompt = _tokenizer().apply_chat_template(
conversation=conversation,
messages=messages,
tools=tools,
tokenize=False,
)
assert '<|DSML|parameter name="command" string="true">view' in prompt
assert '<|DSML|parameter name="path" string="true">/testbed' in prompt
assert 'parameter name="arguments"' not in prompt
@pytest.mark.parametrize("reasoning_effort", ["none", "low", "medium", "high"])
def test_deepseek_v4_accepts_openai_reasoning_effort_values(reasoning_effort):
prompt = _tokenizer().apply_chat_template(
[{"role": "user", "content": "Hello"}],
tokenize=False,
enable_thinking=True,
reasoning_effort=reasoning_effort,
)
assert prompt.endswith("<|Assistant|><think>")
assert "Reasoning Effort: Absolute maximum" not in prompt
def test_deepseek_v4_preserves_reference_max_reasoning_effort():
prompt = _tokenizer().apply_chat_template(
[{"role": "user", "content": "Hello"}],
tokenize=False,
enable_thinking=True,
reasoning_effort="max",
)
assert prompt.startswith(
"<|begin▁of▁sentence|>Reasoning Effort: Absolute maximum"
)
@pytest.mark.parametrize(
("case_id", "kwargs"),
[
(1, {"thinking": True}),
(2, {"thinking": True}),
(3, {"thinking": True}),
(4, {}),
],
)
def test_deepseek_v4_matches_reference_golden_fixtures(case_id, kwargs):
prompt = _render_reference_case(case_id, **kwargs)
expected = (FIXTURES_DIR / f"test_output_{case_id}.txt").read_text()
assert prompt == expected
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for DeepSeekV4ToolParser."""
import json
from unittest.mock import MagicMock
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
MOCK_TOKENIZER = MagicMock()
MOCK_TOKENIZER.get_vocab.return_value = {}
TC_START = "<|DSML|tool_calls>"
TC_END = "</|DSML|tool_calls>"
INV_START = '<|DSML|invoke name="'
INV_END = "</|DSML|invoke>"
PARAM_START = '<|DSML|parameter name="'
PARAM_END = "</|DSML|parameter>"
def make_parser(tools=None) -> DeepSeekV4ToolParser:
return DeepSeekV4ToolParser(MOCK_TOKENIZER, tools=tools)
def make_request(tools=None) -> MagicMock:
req = MagicMock()
req.tools = tools
return req
def build_tool_call(func_name: str, params: dict[str, str]) -> str:
param_strs = "".join(
f'{PARAM_START}{k}" string="true">{v}{PARAM_END}\n' for k, v in params.items()
)
return f'{TC_START}\n{INV_START}{func_name}">\n{param_strs}{INV_END}\n{TC_END}'
def stream(parser: DeepSeekV4ToolParser, full_text: str, chunk_size: int = 7):
deltas = []
previous_text = ""
for start in range(0, len(full_text), chunk_size):
delta_text = full_text[start : start + chunk_size]
current_text = previous_text + delta_text
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[1],
request=make_request(),
)
previous_text = current_text
if delta is not None:
deltas.append(delta)
return deltas
def reconstruct_args(deltas, tool_index: int = 0) -> str:
fragments = []
for delta in deltas:
if delta.tool_calls:
for tool_call in delta.tool_calls:
if (
tool_call.index == tool_index
and tool_call.function
and tool_call.function.arguments
):
fragments.append(tool_call.function.arguments)
return "".join(fragments)
def test_registered():
assert ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4ToolParser
def test_extract_tool_calls():
parser = make_parser()
model_output = "Let me check. " + build_tool_call(
"get_weather", {"location": "Beijing", "unit": "celsius"}
)
result = parser.extract_tool_calls(model_output, make_request())
assert result.tools_called
assert result.content == "Let me check. "
assert len(result.tool_calls) == 1
tool_call = result.tool_calls[0]
assert tool_call.function.name == "get_weather"
assert json.loads(tool_call.function.arguments) == {
"location": "Beijing",
"unit": "celsius",
}
def test_function_calls_block_is_not_accepted():
parser = make_parser()
model_output = build_tool_call("search", {"query": "vllm"}).replace(
"tool_calls", "function_calls"
)
result = parser.extract_tool_calls(model_output, make_request())
assert not result.tools_called
assert result.content == model_output
def test_streaming_extracts_complete_invokes():
parser = make_parser()
full_text = build_tool_call("search", {"query": "deepseek v4"})
deltas = stream(parser, full_text, chunk_size=5)
names = [
tool_call.function.name
for delta in deltas
if delta.tool_calls
for tool_call in delta.tool_calls
]
assert names == ["search"]
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from tests.v1.attention.utils import create_vllm_config
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadataBuilder
from vllm.v1.kv_cache_interface import MLAAttentionSpec
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_storage_block_size():
"""Regression test: DeepseekV4 compression path must compute slot_mapping from
compressed positions, not reuse the uncompressed common metadata mapping.
"""
device = torch.device("cuda")
# storage_block_size = block_size // compress_ratio = 256 // 4 = 64
kv_cache_spec = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
compress_ratio=4,
)
vllm_config = create_vllm_config(max_model_len=1024)
builder = DeepseekV32IndexerMetadataBuilder(
kv_cache_spec=kv_cache_spec,
layer_names=["dummy"],
vllm_config=vllm_config,
device=device,
)
# Construct a single request where:
# - num_computed = 240 (=> compressed_pos_start = 60)
# - query_len = 40 (=> num_groups = 10)
# => compressed positions are 60..69 which cross the storage block boundary at 64.
query_start_loc = torch.tensor([0, 40], dtype=torch.int32, device=device)
query_start_loc_cpu = query_start_loc.cpu()
seq_lens = torch.tensor([280], dtype=torch.int32, device=device) # 240 + 40
# Two blocks: compressed positions 0..63 map to block 5, 64..127 map to block 7.
block_table_tensor = torch.tensor([[5, 7]], dtype=torch.int32, device=device)
# Dummy uncompressed slot mapping (length == uncompressed num_actual_tokens).
slot_mapping = torch.full((40,), -123, dtype=torch.int64, device=device)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens.cpu(),
num_reqs=1,
num_actual_tokens=40,
max_query_len=40,
max_seq_len=280,
block_table_tensor=block_table_tensor,
slot_mapping=slot_mapping,
causal=True,
)
md = builder.build(common_prefix_len=0, common_attn_metadata=common)
# The compressed slot_mapping retains the original uncompressed size (40).
# Only every compress_ratio-th position gets a valid slot; the rest are -1.
assert md.slot_mapping.numel() == 40
valid_slots = md.slot_mapping[md.slot_mapping >= 0]
assert valid_slots.numel() == 10 # 40 tokens / compress_ratio 4
storage_bs = kv_cache_spec.storage_block_size # 64
# Compressed positions 60..63 land in block 5, positions 64..69 in block 7.
expected = torch.tensor(
[
5 * storage_bs + 60,
5 * storage_bs + 61,
5 * storage_bs + 62,
5 * storage_bs + 63,
]
+ [
7 * storage_bs + 0,
7 * storage_bs + 1,
7 * storage_bs + 2,
7 * storage_bs + 3,
7 * storage_bs + 4,
7 * storage_bs + 5,
],
dtype=torch.int64,
device=device,
)
torch.testing.assert_close(valid_slots, expected)
+3 -2
View File
@@ -1855,10 +1855,11 @@ def test_generate_scheduler_kv_cache_config():
def new_mla_spec(cache_dtype_str=None):
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
return MLAAttentionSpec(
block_size=16,
num_kv_heads=16,
head_size=64,
num_kv_heads=1,
head_size=576,
dtype=torch.float32,
cache_dtype_str=cache_dtype_str,
)
+19 -20
View File
@@ -557,19 +557,19 @@ def test_prefill_hybrid_model_eagle():
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
assert len(req1.block_hashes) == num_full_blocks
assert computed_blocks.get_block_ids() == (
[1, 2, 3, 4],
[0, 9, 10, 11],
[0, 16, 17, 18],
[1, 2, 3, 4, 5],
[0, 0, 10, 11, 12],
[0, 0, 17, 18, 19],
)
assert num_computed_tokens == 4 * block_size
assert num_computed_tokens == 5 * block_size
num_new_tokens = len(all_token_ids) - num_computed_tokens
blocks = manager.allocate_slots(
req1, num_new_tokens, num_computed_tokens, computed_blocks
)
assert blocks is not None and blocks.get_block_ids() == (
[22, 23, 24],
[25, 26, 27],
[28, 29, 30],
[22, 23],
[24, 25],
[26, 27],
)
for block_per_group in computed_blocks.blocks:
for block in block_per_group:
@@ -591,7 +591,7 @@ def test_prefill_hybrid_model_eagle():
make_block_hash_with_group_id(block_hashes[0], 1),
make_block_hash_with_group_id(block_hashes[0], 2),
],
4,
5,
)
# Evict the first block of full attention, makes total cache miss.
@@ -605,7 +605,7 @@ def test_prefill_hybrid_model_eagle():
0,
)
# Evict the last block of all layers, reduces the hit length to 3.
# Evict the last block of all layers, reduces the hit length to 4.
_test_partial_request_hit(
manager,
block_size,
@@ -617,10 +617,10 @@ def test_prefill_hybrid_model_eagle():
make_block_hash_with_group_id(block_hashes[-1], 1),
make_block_hash_with_group_id(block_hashes[-1], 2),
],
3,
4,
)
# Evict the last block of full attention, reduces the hit length to 3.
# Evict the last block of full attention, reduces the hit length to 4.
_test_partial_request_hit(
manager,
block_size,
@@ -628,7 +628,7 @@ def test_prefill_hybrid_model_eagle():
"5",
all_token_ids,
[make_block_hash_with_group_id(block_hashes[-1], 0)],
3,
4,
)
# Since the last block of full attention is dropped for eagle, evict
@@ -655,12 +655,11 @@ def test_prefill_hybrid_model_eagle():
3,
)
# Evict different set of blocks for full attention and sliding window makes
# total cache miss.
# The cache hit length of full attention is 4 * block_size.
# The cache hit length of sliding window is 3 * block_size.
# Then it is cache miss as the two type of layers
# have different hit length.
# Evict different set of blocks for full attention and sliding window.
# Full loses its last block so it drops to 4 full blocks after the eagle
# pop; SWA lost block 0 (outside the sliding window of the final hit),
# which is not required for the K+1 anchor at position 4. Coordinated
# single-drop aligns both groups at hit=4.
_test_partial_request_hit(
manager,
block_size,
@@ -672,7 +671,7 @@ def test_prefill_hybrid_model_eagle():
make_block_hash_with_group_id(block_hashes[0], 1),
make_block_hash_with_group_id(block_hashes[0], 2),
],
0,
4,
)
@@ -893,7 +892,7 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
# - 2 groups: 1 full + 1 other
_EAGLE_HYBRID_MODEL_TEST_CASES = [
# 2 groups: 1 full + 1 other
pytest.param(["full", "sliding_window"], 2, id="2g-full+sw"),
pytest.param(["full", "sliding_window"], 3, id="2g-full+sw"),
]
+2
View File
@@ -1892,6 +1892,7 @@ def create_scheduler_with_priority(
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=block_size,
hash_block_size=block_size,
)
@@ -4008,6 +4009,7 @@ def _create_encoder_decoder_scheduler(
vllm_config=vllm_config,
kv_cache_config=kv_cache_config,
block_size=block_size,
hash_block_size=block_size,
structured_output_manager=StructuredOutputManager(vllm_config),
)
@@ -91,8 +91,10 @@ def test_basic_interface():
assert request_id in kv_connector_metadata.reqs_to_recv["my-engine-id"]
req_meta = kv_connector_metadata.reqs_to_recv["my-engine-id"][request_id]
# local_block_ids is list[list[int]] (per-group); flatten for comparison.
all_block_ids = [bid for group in req_meta.local_block_ids for bid in group]
for block_id, block in zip(
req_meta.local_block_ids,
all_block_ids,
scheduler.kv_cache_manager.coordinator.single_type_managers[0].req_to_blocks[
request_id
],
@@ -228,15 +230,15 @@ def test_scheduler_request_finished():
# Case: Capped length (Successful prefill, need to send to decoder)
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
delay_free, _ = scheduler_connector.request_finished(request, block_ids=[10, 11])
delay_free, _ = scheduler_connector.request_finished(request, block_ids=([10, 11],))
assert delay_free is True
assert "id-1" in scheduler_connector._reqs_need_send
assert scheduler_connector._reqs_need_send["id-1"][1] == [10, 11]
assert scheduler_connector._reqs_need_send["id-1"][1] == [[10, 11]]
# Case: Aborted (No need to transfer, free blocks immediately)
scheduler_connector._reqs_need_send.clear()
request.status = RequestStatus.FINISHED_ABORTED
delay_free, _ = scheduler_connector.request_finished(request, block_ids=[12])
delay_free, _ = scheduler_connector.request_finished(request, block_ids=([12],))
assert delay_free is False
assert len(scheduler_connector._reqs_need_send) == 0
assert "id-1" in scheduler_connector._reqs_not_processed
@@ -334,7 +336,7 @@ async def test_kv_producer(monkeypatch):
send_meta = SendBlockMeta(
p_req_id="p-req-1",
transfer_id=transfer_id,
local_block_ids=[10, 11],
local_block_ids=[[10, 11]],
ready=asyncio.Event(),
)
prefill_worker.reqs_need_send[transfer_id] = send_meta
@@ -346,7 +348,7 @@ async def test_kv_producer(monkeypatch):
remote_port=54321,
remote_tp_size=1,
remote_tp_rank=0,
req_blocks={"d-req-1": (transfer_id, [20, 21])},
req_blocks={"d-req-1": (transfer_id, [[20, 21]])},
kv_caches_base_addr=[0x2000],
block_lens=[block_len],
)
@@ -389,7 +391,7 @@ async def test_kv_producer(monkeypatch):
prefill_worker.reqs_need_send[transfer_id] = send_meta
send_meta.sent = 0
send_meta.ready.set()
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [20])
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [[20]])
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# Verify transfer parameters are correct: 11 to 20
@@ -407,7 +409,7 @@ async def test_kv_producer(monkeypatch):
prefill_worker.reqs_need_send[transfer_id] = send_meta
send_meta.sent = 0
send_meta.ready.set()
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [20, 21, 22])
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [[20, 21, 22]])
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# This should not be called because error.
@@ -424,7 +426,7 @@ async def test_kv_producer(monkeypatch):
prefill_worker.reqs_need_send[transfer_id] = send_meta
send_meta.sent = 0
send_meta.ready.clear()
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [20, 21])
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [[20, 21]])
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
# This should not be called because timeout.
@@ -443,7 +445,7 @@ async def test_kv_producer(monkeypatch):
prefill_worker.reqs_need_send[transfer_id] = send_meta
send_meta.sent = 0
send_meta.ready.set()
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [20, 21])
xfer_meta.req_blocks["d-req-1"] = (transfer_id, [[20, 21]])
# Worker processes the consumer's request
await prefill_worker.send_kv_to_decode(identity, mock_socket, xfer_meta)
mock_send_blocks.assert_called_once()
@@ -481,7 +483,7 @@ async def test_kv_consumuer(monkeypatch):
"d-req-1": PullReqMeta(
d_req_id="d-req-1",
transfer_id="xfer-req-1",
local_block_ids=[100, 101],
local_block_ids=[[100, 101]],
remote_engine_id="p-engine",
remote_bootstrap_addr="http://bootstrap:33333",
pull_tasks_count=1,
@@ -514,7 +516,7 @@ async def test_kv_consumuer(monkeypatch):
assert sent_meta.remote_hostname == "127.0.0.1"
assert sent_meta.remote_port == 54321
assert sent_meta.req_blocks["d-req-1"] == ("xfer-req-1", [100, 101])
assert sent_meta.req_blocks["d-req-1"] == ("xfer-req-1", [[100, 101]])
# Verify internal state is updated correctly.
assert "d-req-1" in decode_worker.finished_recving_reqs
@@ -538,7 +540,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
prefill_worker.reqs_need_send["tx-expired"] = SendBlockMeta(
p_req_id="p-req-expired",
transfer_id="tx-expired",
local_block_ids=[1, 2],
local_block_ids=[[1, 2]],
ready=MagicMock(),
expire_time=time.perf_counter() - 100,
)
@@ -547,7 +549,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
prefill_worker.reqs_need_send["tx-active"] = SendBlockMeta(
p_req_id="p-req-active",
transfer_id="tx-active",
local_block_ids=[3, 4],
local_block_ids=[[3, 4]],
ready=MagicMock(),
expire_time=time.perf_counter() + 100,
)
@@ -703,7 +705,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
prefill_worker.sender_loop = asyncio.get_event_loop()
transfer_id = "xfer-hetero-1"
local_block_ids = [10, 11]
local_block_ids = [[10, 11]]
send_meta = SendBlockMeta(
p_req_id="p-req-h1",
transfer_id=transfer_id,
@@ -720,9 +722,9 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
mock_socket.send_multipart = AsyncMock()
identity = b"consumer-hetero"
# Assign different remote block IDs per D rank
# Assign different remote block IDs per D rank (nested per-group)
d_rank_remote_blocks = {
rank: [20 + i * 10, 21 + i * 10] for i, rank in enumerate(target_d_ranks)
rank: [[20 + i * 10, 21 + i * 10]] for i, rank in enumerate(target_d_ranks)
}
with patch.object(
@@ -757,11 +759,15 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
dst_ptrs = call_args[2]
lengths = call_args[3]
# Flatten nested per-group block IDs for assertions
flat_local = [b for g in local_block_ids for b in g]
flat_remote = [b for g in remote_block_ids for b in g]
# Heterogeneous TP: blocks cannot be coalesced because
# local and remote block_lens differ
assert len(src_ptrs) == len(local_block_ids)
assert len(dst_ptrs) == len(local_block_ids)
assert len(lengths) == len(local_block_ids)
assert len(src_ptrs) == len(flat_local)
assert len(dst_ptrs) == len(flat_local)
assert len(lengths) == len(flat_local)
# Compute expected offsets based on TP ratio
if d_tp_size <= P_TP_SIZE:
@@ -775,9 +781,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
expected_dst_off = 0
expected_xfer_len = remote_block_len
for idx, (lblk, rblk) in enumerate(
zip(local_block_ids, remote_block_ids)
):
for idx, (lblk, rblk) in enumerate(zip(flat_local, flat_remote)):
assert src_ptrs[idx] == (
0x1000 + lblk * local_block_len + expected_src_off
)
@@ -0,0 +1,410 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MooncakeConnector HMA (Hybrid Memory Architecture) support.
Covers sliding-window clipping, multi-group metadata shape, multi-group
send trimming, and group-count invariant checking in _build_transfer_params.
"""
import asyncio
from unittest.mock import patch
import pytest
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
KVConnectorRole,
MooncakeConnector,
MooncakeConnectorMetadata,
MooncakeConnectorScheduler,
MooncakeXferMetadata,
SendBlockMeta,
TransferRegion,
)
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
from .utils import create_request, create_vllm_config, make_kv_cache_config
# ---------------------------------------------------------------------------
# test_sw_sizes: blocks_per_sw computed from KVCacheConfig
# ---------------------------------------------------------------------------
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"swa_enabled,expected_blocks_per_sw",
[
# SWA enabled: FullAttentionSpec (0) + SlidingWindowSpec (2048/16=128+1)
(True, [0, 128 + 1]),
# SWA disabled: only FullAttentionSpec (0)
(False, [0]),
],
)
def test_sw_sizes(swa_enabled, expected_blocks_per_sw):
"""blocks_per_sw is correctly computed based on SWA enabled/disabled."""
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_both",
block_size=block_size,
)
# Override so HMA detection works
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled, sw_size=2048
)
scheduler = MooncakeConnectorScheduler(
vllm_config=vllm_config,
engine_id="test-engine",
kv_cache_config=kv_cache_config,
)
assert scheduler.blocks_per_sw == expected_blocks_per_sw
# ---------------------------------------------------------------------------
# test_is_hma_required: derived from kv_cache_config groups
# ---------------------------------------------------------------------------
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"swa_enabled,disable_hma,expected_is_hma",
[
(True, False, True), # SWA group present, HMA enabled
(True, True, False), # SWA group present, but HMA disabled
(False, False, False), # FA only, HMA not needed
],
)
def test_is_hma_required(swa_enabled, disable_hma, expected_is_hma):
"""_is_hma_required is correctly derived from kv_cache_config."""
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_both",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = disable_hma
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=swa_enabled
)
scheduler = MooncakeConnectorScheduler(
vllm_config=vllm_config,
engine_id="test-engine",
kv_cache_config=kv_cache_config,
)
assert scheduler._is_hma_required is expected_is_hma
# ---------------------------------------------------------------------------
# test_get_sw_clipped_blocks: sliding-window clipping logic
# ---------------------------------------------------------------------------
@pytest.mark.cpu_test
def test_get_sw_clipped_blocks():
"""get_sw_clipped_blocks clips SWA group but keeps FA group intact."""
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_both",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
# SW=128 tokens → 128/16 = 8 blocks + 1 = 9 blocks_per_sw
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=True, sw_size=128
)
scheduler = MooncakeConnectorScheduler(
vllm_config=vllm_config,
engine_id="test-engine",
kv_cache_config=kv_cache_config,
)
assert scheduler.blocks_per_sw == [0, 9]
# FA group: 20 blocks, SW group: 20 blocks (exceeds window)
fa_blocks = list(range(20))
sw_blocks = list(range(100, 120))
block_ids = (fa_blocks, sw_blocks)
clipped = scheduler.get_sw_clipped_blocks(block_ids)
# FA: untouched (blocks_per_sw[0] = 0)
assert clipped[0] == fa_blocks
# SW: clipped to last 9 blocks
assert clipped[1] == sw_blocks[-9:]
assert len(clipped[1]) == 9
@pytest.mark.cpu_test
def test_get_sw_clipped_blocks_noop_no_hma():
"""get_sw_clipped_blocks is a no-op when HMA is not required."""
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_both",
block_size=block_size,
)
# FA only → _is_hma_required = False
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=False)
scheduler = MooncakeConnectorScheduler(
vllm_config=vllm_config,
engine_id="test-engine",
kv_cache_config=kv_cache_config,
)
assert scheduler._is_hma_required is False
block_ids = ([1, 2, 3],)
clipped = scheduler.get_sw_clipped_blocks(block_ids)
assert clipped == [[1, 2, 3]]
# ---------------------------------------------------------------------------
# test_metadata_hma_block_ids: MooncakeConnectorMetadata stores per-group IDs
# ---------------------------------------------------------------------------
@pytest.mark.cpu_test
def test_metadata_hma_block_ids():
"""MooncakeConnectorMetadata.add_new_req stores per-group block IDs."""
metadata = MooncakeConnectorMetadata()
# FA group: 6 blocks, SW group: 3 blocks (clipped)
fa_blocks = [0, 1, 2, 3, 4, 5]
sw_blocks = [10, 11, 12]
# Test recv path
metadata.add_new_req(
request_id="recv-req",
local_block_ids=[fa_blocks, sw_blocks],
kv_transfer_params={
"transfer_id": "recv-req",
"remote_engine_id": "remote-engine",
"remote_bootstrap_addr": "http://bootstrap:33333",
},
load_remote_cache=True,
)
assert "recv-req" in metadata.reqs_to_recv["remote-engine"]
req_meta = metadata.reqs_to_recv["remote-engine"]["recv-req"]
assert len(req_meta.local_block_ids) == 2
assert req_meta.local_block_ids[0] == fa_blocks
assert req_meta.local_block_ids[1] == sw_blocks
# Test send path
metadata.add_new_req(
request_id="send-req",
local_block_ids=[fa_blocks, sw_blocks],
kv_transfer_params={
"transfer_id": "send-req",
},
load_remote_cache=False,
)
assert "send-req" in metadata.reqs_to_send
transfer_id, stored_blocks = metadata.reqs_to_send["send-req"]
assert transfer_id == "send-req"
assert len(stored_blocks) == 2
assert stored_blocks[0] == fa_blocks
assert stored_blocks[1] == sw_blocks
# ---------------------------------------------------------------------------
# test_build_transfer_params_multi_group_trimming
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".mooncake_connector.TransferEngine",
FakeMooncakeWrapper,
)
async def test_build_transfer_params_multi_group_trimming(monkeypatch):
"""_build_transfer_params trims per-group blocks when local > remote."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(vllm_config, KVConnectorRole.WORKER)
worker = connector.connector_worker
block_len = 4096
# Call _build_transfer_params directly (avoids send_kv_to_decode
# async event loop complexity).
transfer_id = "xfer-hma-trim"
send_meta = SendBlockMeta(
p_req_id="p-trim",
transfer_id=transfer_id,
# FA: 4 blocks, SW: 3 blocks (producer has more)
local_block_ids=[[10, 11, 12, 13], [20, 21, 22]],
ready=asyncio.Event(),
)
xfer_meta = MooncakeXferMetadata(
remote_hostname="consumer-host",
remote_port=54321,
remote_tp_size=1,
remote_tp_rank=0,
req_blocks={
"d-trim": (
transfer_id,
# FA: 2 blocks, SW: 2 blocks (consumer needs fewer)
[[30, 31], [40, 41]],
)
},
kv_caches_base_addr=[0x2000],
block_lens=[block_len],
)
local_regions = [
TransferRegion(
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
),
]
remote_regions = [
TransferRegion(
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
),
]
ready_reqs = [("d-trim", send_meta)]
(
src_ptrs,
dst_ptrs,
lengths,
err_reqs,
err_msg,
) = await worker._build_transfer_params(
ready_reqs, xfer_meta, local_regions, remote_regions
)
# No errors
assert err_reqs == []
assert err_msg is None
# After trimming: FA [10..13] → last 2 → [12,13]; SW [20..22] → last 2 → [21,22]
# Flattened: [12,13,21,22] = 4 blocks → coalesced into some transfers
assert len(src_ptrs) > 0
assert len(dst_ptrs) == len(src_ptrs)
assert len(lengths) == len(src_ptrs)
worker.shutdown()
# ---------------------------------------------------------------------------
# test_build_transfer_params_group_count_mismatch
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake"
".mooncake_connector.TransferEngine",
FakeMooncakeWrapper,
)
async def test_build_transfer_params_group_count_mismatch(monkeypatch):
"""_build_transfer_params reports an error when group counts differ."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(vllm_config, KVConnectorRole.WORKER)
worker = connector.connector_worker
block_len = 4096
transfer_id = "xfer-mismatch"
send_meta = SendBlockMeta(
p_req_id="p-mismatch",
transfer_id=transfer_id,
# Producer has 2 groups
local_block_ids=[[10, 11], [20, 21]],
ready=asyncio.Event(),
)
# Consumer has only 1 group — group count mismatch
xfer_meta = MooncakeXferMetadata(
remote_hostname="consumer-host",
remote_port=54321,
remote_tp_size=1,
remote_tp_rank=0,
req_blocks={
"d-mismatch": (transfer_id, [[30, 31]]),
},
kv_caches_base_addr=[0x2000],
block_lens=[block_len],
)
local_regions = [
TransferRegion(
base_addr=0x1000, block_len=block_len, kv_block_len=block_len
),
]
remote_regions = [
TransferRegion(
base_addr=0x2000, block_len=block_len, kv_block_len=block_len
),
]
ready_reqs = [("d-mismatch", send_meta)]
(
src_ptrs,
dst_ptrs,
lengths,
err_reqs,
err_msg,
) = await worker._build_transfer_params(
ready_reqs, xfer_meta, local_regions, remote_regions
)
# Mismatched req is reported via err_reqs/err_msg with no transfers built.
assert err_reqs == ["d-mismatch"]
assert err_msg == "KV group count mismatch"
assert src_ptrs == []
assert dst_ptrs == []
assert lengths == []
worker.shutdown()
# ---------------------------------------------------------------------------
# test_request_finished_with_hma_groups
# ---------------------------------------------------------------------------
@pytest.mark.cpu_test
def test_request_finished_with_hma_groups():
"""request_finished correctly handles per-group block_ids."""
block_size = 16
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
block_size=block_size,
)
vllm_config.scheduler_config.disable_hybrid_kv_cache_manager = False
kv_cache_config = make_kv_cache_config(
block_size=block_size, swa_enabled=True, sw_size=128
)
scheduler = MooncakeConnectorScheduler(
vllm_config=vllm_config,
engine_id="test-engine",
kv_cache_config=kv_cache_config,
)
request = create_request(request_id=1, do_remote_decode=True)
request.kv_transfer_params["transfer_id"] = request.request_id
from vllm.v1.request import RequestStatus
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
# 2 groups: FA with 10 blocks, SW with 20 blocks (will be clipped)
fa_blocks = list(range(10))
sw_blocks = list(range(100, 120))
block_ids = (fa_blocks, sw_blocks)
delay_free, _ = scheduler.request_finished(request, block_ids)
assert delay_free is True
assert request.request_id in scheduler._reqs_need_send
_, stored_blocks = scheduler._reqs_need_send[request.request_id]
# FA: untouched
assert stored_blocks[0] == fa_blocks
# SW: clipped to last 9 blocks (sw_size=128, block_size=16 → 8+1=9)
assert stored_blocks[1] == sw_blocks[-9:]
@@ -76,6 +76,7 @@ def create_scheduler() -> Scheduler:
log_stats=True,
structured_output_manager=StructuredOutputManager(vllm_config),
block_size=16,
hash_block_size=16,
)