Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
+37
View File
@@ -3,6 +3,7 @@
import json
import subprocess
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
import numpy as np
@@ -10,6 +11,7 @@ import pytest
from vllm.benchmarks.forward.batch import build_batch
from vllm.benchmarks.forward.main import run_case
from vllm.benchmarks.forward.profile import summarize
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
from vllm.platforms import current_platform
@@ -97,6 +99,35 @@ def test_bench_forward_rejects_oversized_verify(capacity):
run_case(runner, case, SimpleNamespace())
def test_profile_summary():
"""Each replay's leading restore/rendezvous activities are left out, and
kernels are grouped by name without template or function arguments."""
def act(name, ts, dur, cat="kernel"):
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
replay = [
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
("void ns::gemm<128>(Params)", "kernel", 10),
("void ns::gemm<64>(Params)", "kernel", 20),
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
]
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
for r in range(2):
for i, (name, cat, dur) in enumerate(replay):
events.append(act(name, 1000 * r + 100 * i, dur, cat))
summary = summarize(events, replays=2, skip=1)
assert summary["pre_forward_us"] == 5
assert summary["busy_us"] == 34
assert summary["span_us"] == 204
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
("ns::gemm", 30, 2),
("ns::norm", 4, 1),
]
with pytest.raises(ValueError, match="replays"):
summarize(events[:-1], replays=2, skip=1)
@pytest.mark.benchmark
# TODO(woosuk): Support ROCm and re-enable this test there.
@pytest.mark.skipif(
@@ -126,6 +157,8 @@ def test_bench_forward(tmp_path):
"2",
"--output-json",
str(output),
"--profile-dir",
str(tmp_path / "profile"),
]
result = subprocess.run(command, capture_output=True, text=True)
print(result.stdout)
@@ -139,3 +172,7 @@ def test_bench_forward(tmp_path):
assert len(rank0["batch_order"]) == len(row["requests"])
assert rank0["finite"] and rank0["graph_deterministic"]
assert row["median_us"] > 0
profile = rank0["profile"]
assert profile["top_kernels"] and profile["span_us"] > 0
summary = json.loads(Path(profile["summary"]).read_text())
assert Path(summary["trace"]).exists()
+2 -2
View File
@@ -462,8 +462,8 @@ def test_flashinfer_standalone_workspace_size(
@pytest.mark.parametrize(
("node_count", "is_blackwell", "expected"),
[
(1, False, ("trtllm", False)),
(1, True, ("trtllm", False)),
(1, False, ("mnnvl", True)),
(1, True, ("mnnvl", True)),
(2, False, (None, False)),
(2, True, ("mnnvl", False)),
],
@@ -12,8 +12,10 @@ import pytest_asyncio
from tests.utils import RemoteLaunchRenderServer
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
GenerateTokensResponse,
GenerateTokensStreamResponse,
)
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.tokenizers import get_tokenizer
@@ -1579,3 +1581,43 @@ def test_derender_chat_keeps_parser_tool_call_id():
(tool_call,) = choice.message.tool_calls
assert tool_call.id == "functions.get_weather:1"
@pytest.mark.asyncio
@pytest.mark.parametrize("return_token_ids", [None, False, True])
@pytest.mark.parametrize("stream", [False, True])
async def test_derender_completion_preserves_requested_token_ids(
tokenizer, return_token_ids, stream
):
renderer = MagicMock()
renderer.get_tokenizer.return_value = tokenizer
renderer._executor = None
model_config = MagicMock()
model_config.model = MODEL_NAME
model_config.hf_config.model_type = "llama"
derenderer = OnlineDerenderer(
model_config,
renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="string",
)
token_ids = tokenizer.encode("Hello world", add_special_tokens=False)
request = CompletionRequest(
model=MODEL_NAME, prompt="Hi", return_token_ids=return_token_ids
)
payload = _make_generate_response(token_ids, finish_reason="length")
if stream:
response, _ = await derenderer.derender_completion_stream(
MODEL_NAME,
GenerateTokensStreamResponse.model_validate(payload),
completion_request=request,
)
choices = response.choices
else:
choices, _, _ = await derenderer.derender_completion(
[GenerateTokensResponse.model_validate(payload)],
completion_request=request,
)
assert choices[0].text == tokenizer.decode(token_ids)
assert choices[0].token_ids == (token_ids if return_token_ids else None)
@@ -18,13 +18,16 @@ from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
## Parallel Processes Utils
P = ParamSpec("P")
LAUNCH_TIMEOUT_S = 900
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
@dataclasses.dataclass
+41 -30
View File
@@ -147,26 +147,15 @@ def test_fp4_env_defaults():
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
@pytest.mark.parametrize(
(
"use_aiter",
"use_fp4_asm_gemm",
"use_fp4bmm",
),
[
(True, True, True),
(True, True, False),
(True, False, True),
(False, True, True),
],
("use_aiter", "use_fp4bmm"),
[(True, True), (True, False), (False, True)],
)
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
use_aiter,
use_fp4_asm_gemm,
use_fp4bmm,
monkeypatch,
):
@@ -182,22 +171,16 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
_assert_aiter_supported()
on_gfx950_value = on_gfx950()
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
expected_asm_gemm = use_aiter and on_gfx950_value
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv(
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
"1" if use_fp4_asm_gemm else "0",
)
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert (
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
)
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
@@ -695,24 +678,52 @@ def test_asm_fp4_scale_swizzle_rejects_non_2d():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
"""A misaligned weight_scale must disable ASM and take the Triton path."""
@pytest.mark.parametrize(
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
[
(True, True, (1280, 4096), (1280, 256), True),
(True, False, (1280, 4096), (1280, 256), False),
(False, True, (1280, 4096), (1280, 256), False),
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
(True, True, (2560, 160), (2560, 10), False),
],
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
)
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
):
"""A layer takes the ASM layout only when aiter has tuned configs for both
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
Otherwise it stays on the Triton layout."""
from torch.nn.parameter import Parameter
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
monkeypatch.setattr(
rocm_aiter_ops,
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
lambda n, k: preshuffle_tuned,
)
kernel = object.__new__(AiterMxfp4LinearKernel)
kernel.use_asm_gemm = True
kernel.asm_gemm_supported = True
kernel.use_asm_gemm = False
kernel.out_dtype = torch.bfloat16
layer = torch.nn.Module()
layer.weight = Parameter(
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
layer.weight_scale = Parameter(
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
kernel.process_weights_after_loading(layer)
assert kernel.use_asm_gemm is False
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == (10, 2560)
assert layer.weight_scale.is_contiguous()
assert kernel.use_asm_gemm is expected
if not expected:
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
assert layer.weight_scale.is_contiguous()
@@ -0,0 +1,340 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from vllm.platforms import current_platform
HEAD_DIM = 64
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike() or not torch.cuda.is_available(),
reason="Muse Spark kernel requires a CUDA or ROCm GPU",
)
def make_cos_sin_cache(max_pos, rotary_dim, base, device, dtype=torch.float32):
"""The table ``RotaryEmbedding`` builds: cos||sin, fp32 for Muse-Spark."""
inv_freq = 1.0 / (
base
** (
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
/ rotary_dim
)
)
t = torch.arange(max_pos, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j->ij", t, inv_freq)
return torch.cat((freqs.cos(), freqs.sin()), dim=-1).to(dtype)
def qk_norm_rope_ref(x, positions, cos_sin_cache, rotary_dim, eps):
"""fp32 reference for one q or k branch, rounded once; no table means NoPE."""
out = x.float()
out = out * torch.rsqrt(out.pow(2).mean(dim=-1, keepdim=True) + eps)
if cos_sin_cache is not None:
half = rotary_dim // 2
cs = cos_sin_cache[positions].float()
cos = cs[..., :half].unsqueeze(1)
sin = cs[..., half:].unsqueeze(1)
# GPT-J interleaved: the rotated pair is (2i, 2i+1).
x0 = out[..., :rotary_dim:2]
x1 = out[..., 1:rotary_dim:2]
out = out.clone()
out[..., :rotary_dim:2] = x0 * cos - x1 * sin
out[..., 1:rotary_dim:2] = x0 * sin + x1 * cos
return out.to(x.dtype)
def assert_fp8_cache_close(got, expected, exact_fraction=0.999):
"""Compare raw cache codes in the platform's FP8 encoding."""
diff = (got.int() - expected.int()).abs()
within = diff <= 1
assert bool(within.all()), (
f"fp8 cache differs by more than one code in {int((~within).sum())} "
f"of {diff.numel()} elements"
)
exact = float((diff == 0).float().mean())
assert exact >= exact_fraction, (
f"only {exact:.5f} of the fp8 cache is bit-identical to "
f"reshape_and_cache_flash, expected at least {exact_fraction}"
)
def make_case(num_tokens, nq, nkv, dtype, device, with_gate=True, seed=0):
torch.manual_seed(seed)
qsz, kvsz = nq * HEAD_DIM, nkv * HEAD_DIM
gate = qsz if with_gate else 0
qkv = torch.randn(num_tokens, qsz + 2 * kvsz + gate, dtype=dtype, device=device)
return qkv, qsz, kvsz, gate
def split_qkv(qkv, qsz, kvsz, gate):
return qkv.split([qsz, kvsz, kvsz, gate], dim=-1)
def check_qk_norm_rope(
num_tokens,
nq,
nkv,
dtype,
use_rope,
rotary_dim=HEAD_DIM,
with_gate=True,
rope_dtype=torch.float32,
):
device, eps, max_pos = "cuda", 1e-5, 4096
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, with_gate)
qkv_orig = qkv.clone()
cos_sin = (
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device, rope_dtype)
if use_rope
else None
)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
HEAD_DIM,
eps,
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim if use_rope else 0,
)
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
q_ref = qk_norm_rope_ref(
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, qsz)
k_ref = qk_norm_rope_ref(
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, kvsz)
# rsqrtf and lane summation order cost a couple of fp32 ulp, well inside
# one ulp of the stored dtype.
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_rope", [True, False])
def test_qk_norm_rope(dtype, use_rope):
check_qk_norm_rope(17, 32, 4, dtype, use_rope)
@pytest.mark.parametrize(
"num_tokens,variant",
[
pytest.param(64, {"rotary_dim": 32}, id="partial_rotary_dim"),
# A [q | k | v] row (no o-gate) is addressed by its own stride.
pytest.param(33, {"with_gate": False}, id="no_gate_row"),
pytest.param(128, {"rope_dtype": torch.bfloat16}, id="bf16_cos_sin_cache"),
],
)
def test_qk_norm_rope_layout_variants(num_tokens, variant):
check_qk_norm_rope(num_tokens, 32, 4, torch.bfloat16, True, **variant)
# Cover narrow/wide head groups for TP4/TP8 and the one-chunk-per-token case.
@pytest.mark.parametrize(
"num_tokens,nq,nkv,kv_cache_dtype,use_rope",
[
pytest.param(1, 32, 4, "auto", True, id="tp4-decode"),
pytest.param(1, 16, 2, "fp8", False, id="tp8-decode-fp8-nope"),
pytest.param(1025, 32, 4, "auto", True, id="tp4-group2"),
pytest.param(1025, 32, 4, "fp8", True, id="tp4-group2-fp8"),
pytest.param(1537, 32, 4, "auto", True, id="tp4-group4"),
pytest.param(1537, 32, 4, "auto", False, id="tp4-nope"),
pytest.param(3073, 16, 2, "auto", True, id="tp8-group4"),
pytest.param(3073, 16, 2, "fp8", False, id="tp8-fp8-nope"),
pytest.param(40000, 2, 1, "auto", True, id="single-chunk-per-token"),
],
)
def test_kv_insert(num_tokens, nq, nkv, kv_cache_dtype, use_rope):
device, dtype, eps = "cuda", torch.bfloat16, 1e-5
rotary_dim, max_pos, block_size = HEAD_DIM, 4096, 16
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, seed=1)
qkv_orig = qkv.clone()
cos_sin = (
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device) if use_rope else None
)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
storage = torch.uint8 if kv_cache_dtype == "fp8" else dtype
kv_cache = torch.zeros(
num_blocks, nkv, block_size, 2 * HEAD_DIM, dtype=storage, device=device
)
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
if num_tokens > 1:
# -1 marks a token the scheduler did not place; the kernel must skip it
# and leave the cache alone.
slot_mapping[::3] = -1
# Non-unit scales, so a kernel that ignored them would fail loudly.
k_scale = torch.tensor(0.37, dtype=torch.float32, device=device)
v_scale = torch.tensor(1.83, dtype=torch.float32, device=device)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
HEAD_DIM,
eps,
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim if use_rope else 0,
slot_mapping=slot_mapping,
kv_cache=kv_cache,
block_size=block_size,
kv_cache_dtype=kv_cache_dtype,
k_scale=k_scale if kv_cache_dtype != "auto" else None,
v_scale=v_scale if kv_cache_dtype != "auto" else None,
)
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
q_ref = qk_norm_rope_ref(
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, qsz)
k_ref = qk_norm_rope_ref(
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, kvsz)
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
expected = torch.zeros_like(kv_cache)
exp_k, exp_v = expected.transpose(1, 2).split(HEAD_DIM, dim=-1)
ops.reshape_and_cache_flash(
k_out.view(num_tokens, nkv, HEAD_DIM),
v_out.view(num_tokens, nkv, HEAD_DIM),
exp_k,
exp_v,
slot_mapping,
kv_cache_dtype,
k_scale,
v_scale,
)
if kv_cache_dtype == "auto":
torch.testing.assert_close(kv_cache, expected, rtol=0, atol=0)
else:
assert_fp8_cache_close(kv_cache, expected)
# The fused Q output requires E4M3FN; scaled_fp8_quant uses the platform dtype.
@pytest.mark.skipif(
current_platform.is_rocm() and current_platform.fp8_dtype() != torch.float8_e4m3fn,
reason="Fused E4M3FN Q output cannot be compared with ROCm FNUZ quantization",
)
@pytest.mark.parametrize(
"head_dim,num_tokens,rotary_fraction,use_cuda_graph",
[
pytest.param(64, 17, 0, False, id="head64-nope"),
pytest.param(128, 17, 0.5, False, id="head128-partial-rope"),
pytest.param(256, 17, 0, False, id="head256-nope"),
pytest.param(256, 3073, 0.5, False, id="head256-partial-rope-large"),
pytest.param(64, 3073, 0.5, True, id="graph-rope"),
pytest.param(64, 3073, 0, True, id="graph-nope"),
],
)
def test_fused_query_quant_matches_separate_quant(
head_dim, num_tokens, rotary_fraction, use_cuda_graph
):
"""Preserve the rounding boundary, scales, padded Q, and KV/gate writes."""
torch.manual_seed(42)
nq, nkv, block_size = 16, 2, 16
qsz, kvsz = nq * head_dim, nkv * head_dim
qkv = torch.randn(num_tokens, 2 * (qsz + kvsz), device="cuda", dtype=torch.bfloat16)
original = qkv.clone()
reference = qkv.clone()
q_fp8 = torch.empty(num_tokens, qsz, device="cuda", dtype=torch.float8_e4m3fn)
positions = torch.arange(num_tokens, device="cuda", dtype=torch.int64)
rotary_dim = int(head_dim * rotary_fraction)
cos_sin = (
make_cos_sin_cache(num_tokens, rotary_dim, 500_000.0, "cuda")
if rotary_dim
else None
)
cache = torch.zeros(
(num_tokens + block_size - 1) // block_size,
nkv,
block_size,
2 * head_dim,
device="cuda",
dtype=torch.uint8,
)
expected_cache = torch.zeros_like(cache)
slots = positions.clone()
slots[::3] = -1
# Small enough that part of Q saturates the e4m3 range.
scale = torch.tensor(0.001, device="cuda", dtype=torch.float32)
kwargs = dict(
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim,
slot_mapping=slots,
block_size=block_size,
kv_cache_dtype="fp8",
k_scale=torch.tensor(0.37, device="cuda"),
v_scale=torch.tensor(1.83, device="cuda"),
)
ops.fused_muse_spark_qknorm_rope_kv_insert(
reference,
positions,
nq,
nkv,
head_dim,
1e-5,
kv_cache=expected_cache,
**kwargs,
)
expected_q, _ = ops.scaled_fp8_quant(reference[:, :qsz], scale=scale)
def run():
qkv.copy_(original)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
head_dim,
1e-5,
kv_cache=cache,
q_fp8_out=q_fp8,
q_scale=scale,
**kwargs,
)
run()
if use_cuda_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
qkv.zero_()
cache.zero_()
q_fp8.zero_()
graph.replay()
torch.testing.assert_close(
q_fp8.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0
)
torch.testing.assert_close(cache, expected_cache, rtol=0, atol=0)
torch.testing.assert_close(qkv[:, :qsz], original[:, :qsz], rtol=0, atol=0)
torch.testing.assert_close(qkv[:, qsz:], reference[:, qsz:], rtol=0, atol=0)
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weighted RMSNorm + partial NeoX RoPE with separate paged K/V caches."""
import pytest
import torch
import vllm._custom_ops as ops
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not torch.cuda.is_available(),
reason="Step-5 kernel is only built for CUDA",
)
def norm_rope_reference(x, weight, cos_sin, positions, rotary_dim, eps, bias):
xf = x.float()
normed = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps)
out = (normed * (weight.float() + bias)).to(x.dtype)
half = rotary_dim // 2
cos, sin = cos_sin[positions].float().unsqueeze(1).chunk(2, dim=-1)
x0, x1 = out[..., :half].float(), out[..., half:rotary_dim].float()
out[..., :half] = x0 * cos - x1 * sin
out[..., half:rotary_dim] = x0 * sin + x1 * cos
return out
def check_norm_rope_and_cache(
dtype=torch.bfloat16,
kv_cache_dtype="auto",
head_dim=128,
rotary_dim=64,
num_tokens=19,
block_size=16,
fp32_weights=False,
bias=0.0,
strided_cache=False,
use_cuda_graph=False,
):
"""Cover rounding, 192-wide RoPE, padded rows and short/negative slots."""
torch.manual_seed(0)
nq, nkv, max_pos, eps = 5, 2, 64, 1e-5
width = (nq + 2 * nkv) * head_dim
storage = torch.randn(num_tokens, width + 8, device="cuda", dtype=dtype)
qkv = storage[:, :width]
original = storage.clone()
weight_dtype = torch.float32 if fp32_weights else dtype
qw = torch.randn(head_dim, device="cuda", dtype=weight_dtype)
kw = torch.randn_like(qw)
angles = torch.randn(max_pos, rotary_dim // 2, device="cuda")
rope_dtype = torch.float32 if fp32_weights else dtype
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(rope_dtype)
positions = torch.randint(max_pos, (num_tokens,), device="cuda")
num_blocks = (num_tokens + block_size - 1) // block_size + 1
cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
shape = (num_blocks, block_size, nkv, head_dim)
if strided_cache:
shape = (num_blocks, nkv, block_size, head_dim)
key_cache = torch.zeros(shape, device="cuda", dtype=cache_dtype)
value_cache = torch.zeros_like(key_cache)
if strided_cache:
key_cache = key_cache.transpose(1, 2)
value_cache = value_cache.transpose(1, 2)
# A short mapping models graph padding; negative slots must stay unwritten.
num_slots = max(0, num_tokens - 3) if num_tokens > 1 else num_tokens
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_slots]
slots[1::3] = -1
k_scale = torch.tensor(0.37, device="cuda")
v_scale = torch.tensor(1.83, device="cuda")
def run():
return ops.fused_qknorm_rope_cache(
qkv,
qw,
kw,
cos_sin,
positions,
slots,
key_cache,
value_cache,
nq,
nkv,
head_dim,
rotary_dim,
eps,
norm_weight_bias=bias,
kv_cache_dtype=kv_cache_dtype,
k_scale=k_scale,
v_scale=v_scale,
)
q, k, v = run()
if use_cuda_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
q, k, v = run()
key_cache.zero_()
value_cache.zero_()
graph.replay()
q_in, k_in, v_in = qkv.split(
(nq * head_dim, nkv * head_dim, nkv * head_dim), dim=-1
)
tol = 1e-2 if dtype == torch.bfloat16 else 2e-3
for actual, src, weight, heads in ((q, q_in, qw, nq), (k, k_in, kw, nkv)):
expected = norm_rope_reference(
src.reshape(num_tokens, heads, head_dim),
weight,
cos_sin,
positions,
rotary_dim,
eps,
bias,
).reshape_as(actual)
torch.testing.assert_close(actual, expected, rtol=tol, atol=tol)
torch.testing.assert_close(v, v_in, rtol=0, atol=0)
torch.testing.assert_close(storage, original, rtol=0, atol=0)
expected_k, expected_v = torch.zeros_like(key_cache), torch.zeros_like(value_cache)
if num_slots:
ops.reshape_and_cache_flash(
k.reshape(num_tokens, nkv, head_dim),
v.reshape(num_tokens, nkv, head_dim),
expected_k,
expected_v,
slots,
kv_cache_dtype,
k_scale,
v_scale,
)
for actual, expected in ((key_cache, expected_k), (value_cache, expected_v)):
if kv_cache_dtype == "fp8":
# Reciprocal multiplication versus division may differ by one code.
assert (actual.int() - expected.int()).abs().max() <= 1
else:
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
"case",
[
pytest.param(dict(head_dim=64, rotary_dim=32, num_tokens=1), id="decode"),
pytest.param(
dict(block_size=7, fp32_weights=True, bias=1.0, strided_cache=True),
id="strided-cache-fp32-weights",
),
pytest.param(
dict(head_dim=192, rotary_dim=128, fp32_weights=True),
id="head192-partial-rope",
),
pytest.param(
dict(
head_dim=192,
rotary_dim=192,
block_size=7,
bias=1.0,
strided_cache=True,
kv_cache_dtype="fp8",
),
id="head192-full-rope-fp8",
),
pytest.param(
dict(
head_dim=256, rotary_dim=256, strided_cache=True, kv_cache_dtype="fp8"
),
id="head256-full-rope-fp8",
),
pytest.param(
dict(head_dim=192, rotary_dim=0, fp32_weights=True, bias=1.0),
id="no-rope",
),
pytest.param(
dict(head_dim=64, rotary_dim=8, fp32_weights=True),
id="minimum-rotary-dim",
),
],
)
def test_norm_rope_and_cache(dtype, case):
check_norm_rope_and_cache(dtype=dtype, **case)
@pytest.mark.parametrize(
"case",
[
pytest.param(dict(num_tokens=0), id="empty-input"),
pytest.param(dict(use_cuda_graph=True), id="graph-auto-cache"),
pytest.param(
dict(use_cuda_graph=True, kv_cache_dtype="fp8"), id="graph-fp8-cache"
),
],
)
def test_empty_input_and_graph_replay(case):
check_norm_rope_and_cache(**case)
+218
View File
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the LoRA-aware FlashInfer TRTLLM MXFP4 experts
(`TrtLlmMxfp4LoRAExperts`) against a float64 reference."""
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmMxfp4LoRAExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_requested_kernel_cls,
backend_to_kernel_cls,
convert_weight_to_mxfp4_moe_kernel_format,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
from vllm.utils.torch_utils import set_random_seed
# FlashInfer's TRTLLM MoE LoRA kernels only run on SM100-family GPUs.
pytestmark = [
pytest.mark.skipif(
not current_platform.is_device_capability_family(100)
or not has_flashinfer_trtllm_fused_moe(),
reason="Requires an SM100-family GPU and FlashInfer TRTLLM fused MoE",
),
pytest.mark.skip_global_cleanup,
]
E, N, K, T, TOPK = 16, 256, 512, 33, 4
E2M1 = [0, 0.5, 1, 1.5, 2, 3, 4, 6]
def _dequant_mxfp4(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
"""(E, R, K/2) packed nibbles + (E, R, K/32) e8m0 scales -> (E, R, K) f64."""
nibbles = torch.stack([weight & 0xF, weight >> 4], dim=-1).flatten(-2).long()
magnitude = torch.tensor(E2M1, dtype=torch.float64, device=weight.device)
values = magnitude[nibbles & 7] * torch.where(nibbles >> 3 == 1, -1.0, 1.0)
block_scale = torch.pow(2.0, scale.double() - 127.0)
return values * block_scale.repeat_interleave(32, dim=-1)
def _make_experts(has_bias: bool, activation=MoEActivation.SILU):
w13 = torch.randint(0, 256, (E, 2 * N, K // 2), dtype=torch.uint8, device="cuda")
w2 = torch.randint(0, 256, (E, K, N // 2), dtype=torch.uint8, device="cuda")
s13 = torch.randint(118, 122, (E, 2 * N, K // 32), dtype=torch.uint8, device="cuda")
s2 = torch.randint(118, 122, (E, K, N // 32), dtype=torch.uint8, device="cuda")
b13 = torch.randn(E, 2 * N, device="cuda") / 4 if has_bias else None
b2 = torch.randn(E, K, device="cuda") / 4 if has_bias else None
k13, k2, ks13, ks2, kb13, kb2 = convert_weight_to_mxfp4_moe_kernel_format(
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
None,
w13.clone(),
w2.clone(),
s13.clone(),
s2.clone(),
None if b13 is None else b13.clone(),
None if b2 is None else b2.clone(),
_cache_permute_indices={},
)
config = make_dummy_moe_config(
num_experts=E,
experts_per_token=TOPK,
hidden_dim=K,
intermediate_size=N,
activation=activation,
)
quant_config = mxfp4_w4a16_moe_quant_config(ks13, ks2, kb13, kb2)
experts = TrtLlmMxfp4LoRAExperts(config, quant_config)
reference = (_dequant_mxfp4(w13, s13), _dequant_mxfp4(w2, s2), b13, b2)
return experts, (k13, k2), reference
def _make_inputs():
x = torch.randn(T, K, device="cuda").to(torch.bfloat16)
ids = torch.stack([torch.randperm(E, device="cuda")[:TOPK] for _ in range(T)])
weights = torch.rand(T, TOPK, device="cuda", dtype=torch.float32) + 0.1
return x, ids.to(torch.int32), weights
def _reference(reference, x, ids, weights, gate_delta=None, up_delta=None):
"""Float64 MoE; the checkpoint w13 layout is [gate | up]."""
w13, w2, b13, b2 = reference
ids = ids.long()
h = torch.einsum("tk,tjnk->tjn", x.double(), w13[ids])
if b13 is not None:
h = h + b13.double()[ids]
gate, up = h[..., :N], h[..., N:]
if gate_delta is not None:
gate, up = gate + gate_delta.double(), up + up_delta.double()
act = torch.nn.functional.silu(gate) * up
out = torch.einsum("tjn,tjkn->tjk", act, w2[ids])
if b2 is not None:
out = out + b2.double()[ids]
return act, (out * weights.double()[..., None]).sum(1)
def _run_lora_path(experts, kernel_weights, x, ids, weights, delta):
"""`delta` is (T, TOPK, 2I) in the [up | gate] order of apply_w13_lora."""
lora_context = SimpleNamespace(w13_num_slices=2)
gemm2, expert_weights, idx_map, act_permuted = experts.invoke_routed_moe(
hidden_states=x,
w1=kernel_weights[0],
w2=kernel_weights[1],
topk_ids_and_weights=(ids, weights),
gemm1_lora_delta=experts.prepare_gemm1_lora_delta(delta, lora_context, ids),
global_num_experts=E,
a1q_scale=None,
output=torch.empty(T, K, dtype=torch.bfloat16, device="cuda"),
)
assert act_permuted.dtype == torch.bfloat16
act = experts._unpermute_activation(act_permuted, idx_map, T, TOPK, N)
output = torch.empty(T, K, dtype=torch.bfloat16, device="cuda")
no_w2_delta = torch.zeros(T, TOPK, K, dtype=torch.bfloat16, device="cuda")
experts._finalize_with_w2_lora(
output, gemm2, expert_weights, idx_map, no_w2_delta, T, TOPK
)
return act.view(T, TOPK, N), output
def _rel_err(actual: torch.Tensor, expected: torch.Tensor) -> float:
actual, expected = actual.double(), expected.double()
return float((actual - expected).norm() / expected.norm())
@pytest.mark.parametrize("has_bias", [False, True])
@torch.inference_mode()
def test_gemm1_lora_delta_matches_reference(has_bias: bool):
set_random_seed(0)
experts, kernel_weights, reference = _make_experts(has_bias)
x, ids, weights = _make_inputs()
gate_delta = (torch.randn(T, TOPK, N, device="cuda") / 2).to(torch.bfloat16)
up_delta = (torch.randn(T, TOPK, N, device="cuda") / 2).to(torch.bfloat16)
ref_act, ref_out = _reference(reference, x, ids, weights, gate_delta, up_delta)
_, ref_base_out = _reference(reference, x, ids, weights)
# The delta has to matter for the layout check below to mean anything.
assert _rel_err(ref_out, ref_base_out) > 0.3
delta = torch.cat([up_delta, gate_delta], dim=-1)
act, out = _run_lora_path(experts, kernel_weights, x, ids, weights, delta)
assert _rel_err(act, ref_act) < 0.02
assert _rel_err(out, ref_out) < 0.02
swapped = torch.cat([gate_delta, up_delta], dim=-1)
_, out = _run_lora_path(experts, kernel_weights, x, ids, weights, swapped)
assert _rel_err(out, ref_out) > 0.3
@pytest.mark.parametrize("has_bias", [False, True])
@torch.inference_mode()
def test_zero_delta_matches_base_path(has_bias: bool):
set_random_seed(0)
experts, kernel_weights, reference = _make_experts(has_bias)
x, ids, weights = _make_inputs()
base = torch.empty(T, K, dtype=torch.bfloat16, device="cuda")
experts.invoke_routed_moe(
hidden_states=x,
w1=kernel_weights[0],
w2=kernel_weights[1],
topk_ids_and_weights=(ids, weights),
gemm1_lora_delta=None,
global_num_experts=E,
a1q_scale=None,
output=base,
)
_, ref_out = _reference(reference, x, ids, weights)
assert _rel_err(base, ref_out) < 0.02
zero = torch.zeros(T, TOPK, 2 * N, dtype=torch.bfloat16, device="cuda")
_, out = _run_lora_path(experts, kernel_weights, x, ids, weights, zero)
assert _rel_err(out, base) < 0.02
@pytest.mark.parametrize("w13_num_slices", [1, 2])
@pytest.mark.parametrize("activation", [MoEActivation.SILU, MoEActivation.SWIGLUOAI])
@torch.inference_mode()
def test_prepare_gemm1_lora_delta_layout(activation, w13_num_slices: int):
set_random_seed(0)
experts, _, _ = _make_experts(False, activation)
ids = _make_inputs()[1]
gate = torch.randn(T, TOPK, N, device="cuda").to(torch.bfloat16)
up = torch.randn(T, TOPK, N, device="cuda").to(torch.bfloat16)
if w13_num_slices == 2:
# What apply_w13_lora(swap_w13_slices=True) writes for per-expert LoRA.
pair, halves = (up, gate), (up, gate)
else:
# A fused (3D) adapter follows the checkpoint w13 row order.
pair, halves = (gate, up), (gate, up)
if activation == MoEActivation.SWIGLUOAI:
delta = torch.stack(pair, dim=-1).flatten(-2)
else:
delta = torch.cat(halves, dim=-1)
lora_context = SimpleNamespace(w13_num_slices=w13_num_slices)
prepared = experts.prepare_gemm1_lora_delta(delta, lora_context, ids)
assert torch.equal(prepared, torch.cat([up, gate], dim=-1))
def test_lora_experts_are_opt_in_for_bf16_activations():
config = make_dummy_moe_config(num_experts=E, hidden_dim=K, intermediate_size=N)
bf16 = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16
mxfp8 = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
assert _requested_kernel_cls(bf16, config) == backend_to_kernel_cls(bf16)
assert _requested_kernel_cls(mxfp8, config) == backend_to_kernel_cls(mxfp8)
config.is_lora_enabled = True
assert _requested_kernel_cls(bf16, config)[0] is TrtLlmMxfp4LoRAExperts
with pytest.raises(ValueError, match="MXFP8 activations"):
_requested_kernel_cls(mxfp8, config)
marlin = Mxfp4MoeBackend.MARLIN
assert _requested_kernel_cls(marlin, config) == backend_to_kernel_cls(marlin)
@@ -110,7 +110,7 @@ def test_modernbert_models(
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from types import SimpleNamespace
import numpy as np
import pytest
@@ -17,6 +18,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFieldConfig
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
from ....conftest import ImageTestAssets
from ...utils import build_model_context
@@ -348,3 +350,56 @@ def test_encoder_chunk_no_free_memory_falls_back_to_one():
)
== 1
)
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
model = object.__new__(Gemma4ForConditionalGeneration)
vision_config = SimpleNamespace(
patch_size=16, pooling_kernel_size=3, default_output_length=280
)
model.vision_tower = SimpleNamespace(config=vision_config)
model.config = SimpleNamespace(vision_config=vision_config)
return model
@pytest.mark.parametrize(
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
)
def test_encoder_cudagraph_capture_rows_fit_token_budget(
token_budget: int, row_tokens: int
):
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
model = _gemma4_for_encoder_cudagraph()
inputs = model.prepare_encoder_cudagraph_capture_inputs(
token_budget=token_budget,
max_batch_size=58,
max_frames_per_batch=58 * 32,
device="meta",
dtype=torch.bfloat16,
axis_keys=(row_tokens,),
)
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
assert row_patches == row_tokens * 9
assert num_rows == max(1, token_budget // row_tokens)
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
model = _gemma4_for_encoder_cudagraph()
mm_kwargs = {
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
"pixel_position_ids": [
torch.zeros(2520, 2, dtype=torch.long),
torch.zeros(630, 2, dtype=torch.long),
],
}
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
assert [spec.output_tokens for spec in specs] == [280, 70]
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The DeepSeek-V4/V4.1 VL wrappers stream the language model's weights to it
as one contiguous group."""
import pytest
import torch
from torch import nn
from vllm.model_executor.models.utils import WeightsMapper
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA implementations"
)
class _FakeFinalizingLanguageModel(nn.Module):
finalizes_weights_during_load = True
def __init__(self) -> None:
super().__init__()
self.tensor_a = nn.Parameter(torch.zeros(1))
self.tensor_c = nn.Parameter(torch.zeros(1))
self.load_calls = 0
self.finalized_values: list[tuple[float, float]] = []
def load_weights(self, weights) -> set[str]:
self.load_calls += 1
loaded = set()
for name, value in weights:
getattr(self, name).data.copy_(value)
loaded.add(name)
self.finalized_values.append((self.tensor_a.item(), self.tensor_c.item()))
return loaded
def _v4_wrapper():
from vllm.models.deepseek_v4.common.vl_model import (
DeepseekV4ForConditionalGeneration,
)
return DeepseekV4ForConditionalGeneration
def _v41_wrapper():
from vllm.models.deepseek_v41.nvidia.vl_model import DeepseekV41ForCausalLM
return DeepseekV41ForCausalLM
@pytest.mark.parametrize("wrapper", [_v4_wrapper, _v41_wrapper], ids=["v4", "v41"])
def test_load_weights_streams_language_model_group(wrapper) -> None:
model = object.__new__(wrapper())
nn.Module.__init__(model)
model.language_model = _FakeFinalizingLanguageModel()
model.vision = nn.Module()
model.vision.tensor_b = nn.Parameter(torch.zeros(1))
model.hf_to_vllm_mapper = WeightsMapper()
def interleaved_weights():
yield "language_model.tensor_a", torch.tensor([1.0])
# Loaded before the stream is drained, not after.
assert model.language_model.tensor_a.item() == 1.0
# Splits the language model's weights unless the wrapper regroups them.
yield "vision.tensor_b", torch.tensor([2.0])
yield "language_model.tensor_c", torch.tensor([3.0])
loaded = model.load_weights(interleaved_weights())
assert loaded == {
"language_model.tensor_a",
"vision.tensor_b",
"language_model.tensor_c",
}
assert model.language_model.load_calls == 1
assert model.language_model.finalized_values == [(1.0, 3.0)]
assert model.vision.tensor_b.item() == 2.0
+131
View File
@@ -6,6 +6,7 @@ These validate that the engine-driven parser correctly handles
Qwen3 XML-style tool calls.
"""
import dataclasses
import json
from unittest.mock import MagicMock
@@ -808,6 +809,136 @@ class TestArgConverter:
assert result == {"city": "Tokyo", "expr": "x<5"}
WRITE_CALL_HEAD = (
"<tool_call>\n<function=write_file>\n"
"<parameter=path>\n/repo/app.py\n</parameter>\n"
"<parameter=mode>\n644\n</parameter>\n"
"<parameter=content>\n"
)
WRITE_CALL_TAIL = "\n</parameter>\n</function>\n</tool_call>"
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
# the value, including prefixes of the closing and next-parameter tags.
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
class TestArgValueExtender:
"""The open parameter value streams without re-running the converter."""
@pytest.fixture
def write_tool(self):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
return ChatCompletionToolsParam(
type="function",
function={
"name": "write_file",
"parameters": {
"type": "object",
"properties": {
"path": {"type": "string"},
"mode": {"type": "integer"},
"content": {"type": "string"},
},
},
},
)
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
@pytest.mark.parametrize("parser_name", ["qwen3", "mimo"])
def test_long_call_streams_non_streaming_arguments(
self, mock_tokenizer, mock_request, write_tool, use_schema, parser_name
):
from vllm.parser.mimo import MiMoParser
from vllm.parser.qwen3 import Qwen3Parser
cls = MiMoParser if parser_name == "mimo" else Qwen3Parser
mock_request.tools = [write_tool] if use_schema else []
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
def make_parser():
return cls(mock_tokenizer, chat_template_kwargs={"enable_thinking": False})
results = simulate_tool_streaming(
make_parser(),
mock_request,
[text[i : i + 7] for i in range(0, len(text), 7)],
)
result = make_parser().extract_tool_calls(text, mock_request)
assert collect_tool_arguments(results) == (
result.tool_calls[0].function.arguments
)
# MiMo keeps the newlines around the value; the schema coerces both.
mode = '"\\n644\\n"' if parser_name == "mimo" else '"644"'
if use_schema:
mode = "644"
assert f'"mode": {mode},' in collect_tool_arguments(results)
def test_conversions_do_not_grow_with_argument_length(
self, mock_tokenizer, mock_request, write_tool
):
mock_request.tools = [write_tool]
base = qwen3_config(thinking=False)
converter = base.arg_converter
assert converter is not None
def streaming_conversions(lines: int) -> int:
calls = 0
def counting_converter(raw_args: str, partial: bool) -> str:
nonlocal calls
calls += partial
return converter(raw_args, partial)
parser = ParserEngine(
mock_tokenizer,
parser_engine_config=dataclasses.replace(
base, arg_converter=counting_converter
),
)
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
simulate_tool_streaming(
parser,
mock_request,
[text[i : i + 4] for i in range(0, len(text), 4)],
)
return calls
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
def test_extender_follows_newline_trimming_and_split_end(self):
from vllm.parser.qwen3 import _qwen3_arg_value_extender
extend = _qwen3_arg_value_extender("<parameter=k>")
assert extend is not None
deltas = ["\n", 'a"\n', "b", "\n", "</param", "eter>"]
expected = ["", 'a\\"', "\\nb", "", "\\n</param", None]
assert [extend(d) for d in deltas] == expected
def test_extender_stops_at_split_next_parameter(self):
from vllm.parser.qwen3 import _qwen3_arg_value_extender
extend = _qwen3_arg_value_extender("<parameter=k>x")
assert extend is not None
assert [extend(d) for d in ["< para", "meter ="]] == ["< para", None]
@pytest.mark.parametrize(
"raw_args",
[
"<parameter=k>x</parameter>",
"<parameter=k",
"<parameter=k>1</parameter><parameter=j>2</parameter><parameter=k>3",
],
ids=["closed_value", "open_name", "repeated_key_not_last"],
)
def test_no_extender_unless_open_value_is_last(self, raw_args):
from vllm.parser.qwen3 import _qwen3_arg_value_extender
assert _qwen3_arg_value_extender(raw_args) is None
class TestSchemaAwareTypeCoercion:
"""Verify that _fix_arg_types corrects miscoerced values using the
tool schema."""
+98 -2
View File
@@ -2,13 +2,20 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Sequence
from types import SimpleNamespace
from typing import Any, cast
from unittest.mock import MagicMock
import pytest
import torch
from vllm import CompletionOutput, PromptType, RequestOutput, SamplingParams
from vllm import (
CompletionOutput,
PromptType,
RequestOutput,
SamplingParams,
TokensPrompt,
)
from vllm import logger as vllm_logger
from vllm.config import VllmConfig
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
@@ -17,9 +24,10 @@ from vllm.entrypoints.generate.beam_search.utils import (
BeamSearchInstance,
BeamSearchSequence,
)
from vllm.exceptions import VLLMValidationError
from vllm.inputs import EngineInput, TokensInput
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.sampling_params import BeamSearchParams
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
from vllm.v1.structured_output.backend_types import StructuredOutputBackend
@@ -48,8 +56,10 @@ class _Renderer:
class _VllmConfig:
use_v2_model_runner = True
watermark_config = object()
speculative_config = None
model_config = SimpleNamespace(is_diffusion=False)
def _check_supports_watermarking(self, params: BeamSearchParams) -> bool:
return VllmConfig._check_supports_watermarking(cast(VllmConfig, self), params)
@@ -322,3 +332,89 @@ def test_offline_structured_beam_search_disables_internal_watermarking() -> None
assert beam_params.watermarking is False
assert beam_params.allowed_token_ids == [0]
assert base_params.allowed_token_ids is None
@pytest.mark.asyncio
@pytest.mark.parametrize("offline", [False, True])
@pytest.mark.parametrize("exhausted", [False, True])
async def test_explicit_beam_allowlist_uses_exact_scores(offline, exhausted):
prompt: TokensInput = {"type": "token", "prompt_token_ids": [1]}
base = SamplingParams(watermarking=False)
template = [r async for r in _EngineClient().generate(prompt, base)][0]
template.outputs[0].logprobs[0] = dict(
reversed(list(template.outputs[0].logprobs[0].items()))
)
calls = []
if exhausted:
for logprob in template.outputs[0].logprobs[0].values():
logprob.logprob = float("-inf")
def score(params):
assert params.logprob_token_ids == [13, 14]
assert params.logprobs is None and params.allowed_token_ids is None
calls.append(params)
return template
class Client(_EngineClient):
async def generate(self, prompt, params, *args, **kwargs):
yield score(params)
class Offline(_OfflineServing):
_beam_search_step = BeamSearchOfflineMixin._beam_search_step
def _render_and_run_requests(self, prompts, params, output_type, **kwargs):
return [score(p) for p in params]
params = BeamSearchParams(beam_width=2, max_tokens=2, allowed_token_ids=[13, 14])
ids: list[Sequence[int]]
scores: list[float | None]
if offline:
result = (
Offline().beam_search([cast(TokensPrompt, prompt)], params)[0].sequences
)
ids = [b.tokens[1:] for b in result]
scores = [b.cum_logprob for b in result]
step_scores = [step for b in result for step in b.logprobs]
else:
serving = _AsyncServing()
serving.engine_client = Client()
online_result = [r async for r in serving.beam_search(prompt, "test", params)][
0
]
ids = [b.token_ids for b in online_result.outputs]
scores = [b.cumulative_logprob for b in online_result.outputs]
step_scores = [
step for b in online_result.outputs for step in (b.logprobs or [])
]
if exhausted:
assert len(calls) == 1
assert ids == scores == []
return
assert len(calls) == 3
assert ids[0] == [13, 13] and all(set(x) <= {13, 14} for x in ids)
assert scores == [-6.0, -7.0]
assert all(list(step) == [13, 14] for step in step_scores)
assert all(score.rank is None for step in step_scores for score in step.values())
@pytest.mark.parametrize(
"case", ["empty", "grammar", "spec", "diffusion", "v1", "size", "id"]
)
def test_beam_allowlist_rejects_unsupported_requests(case):
config = SimpleNamespace(
use_v2_model_runner=case != "v1",
speculative_config=object() if case == "spec" else None,
model_config=SimpleNamespace(
is_diffusion=case == "diffusion",
max_logprobs=20,
get_vocab_size=lambda: 256,
),
)
ids = {"empty": [], "size": list(range(129)), "id": [256]}.get(case, [13])
params = BeamSearchParams(2, 1, allowed_token_ids=ids)
if case == "grammar":
params.structured_outputs = StructuredOutputsParams(regex="a")
sampling = SamplingParams(logprobs=4)
with pytest.raises((ValueError, VLLMValidationError)):
params.configure_logprobs(sampling, config)
sampling._validate_logprobs(config.model_config)
+82 -228
View File
@@ -58,7 +58,6 @@ from vllm.v1.kv_cache_interface import (
SlidingWindowSpec,
SparseCacheRole,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
@@ -1945,247 +1944,102 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
)
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
is_block_outermost=True
)
),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
speculative_config=SimpleNamespace(
method=method,
use_eagle=lambda: True,
use_eagle_block_drop=lambda: True,
),
def _spec_decode_kv_cache_groups(worker_specs, use_eagle_block_drop=True):
config = VllmConfig(model_config=ModelConfig(max_model_len=64))
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
method="dflash",
use_eagle=lambda: True,
use_eagle_block_drop=lambda: use_eagle_block_drop,
use_multi_module_mtp=lambda: False,
parallel_drafting=False,
)
configs = get_kv_cache_configs(config, worker_specs, [2**30] * len(worker_specs))
return generate_scheduler_kv_cache_config(configs).kv_cache_groups
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
def _as_draft(spec):
return replace(spec, is_draft=True)
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to match that dtype while retaining the draft
marker as a group boundary.
"""
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
specs = {
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
}
def _hybrid_specs(draft=True):
"""Qwen3.5-shaped [GDN x3, full-attn] x2, plus an MTP full-attn layer."""
specs = {}
for i in range(8):
specs[f"layers.{i}"] = (
new_kv_cache_spec()
if i % 4 == 3
else new_mamba_spec(mamba_cache_mode="align")
)
if draft:
draft_spec = new_mla_spec(block_size=64)
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
specs["mtp.0"] = _as_draft(new_kv_cache_spec())
return specs
def test_draft_group_annotated_on_hybrid_general_path():
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
# is identifiable without keying off a model version.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "draft.attn.0" in flagged[0].layer_names
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
# Packed uniform-type groups can contain distinct target and draft layer
# specs; the combined group still holds volatile draft KV and must be
# flagged. What must never happen is a Mamba group being flagged: that
# widens its lookup window to two consecutive chunks, which align-mode
# checkpointing never produces, zeroing every lookup.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(),
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
)
for group in groups:
if "draft.attn.0" in group.layer_names:
assert group.is_eagle_group
if isinstance(group.kv_cache_spec, MambaSpec):
assert not group.is_eagle_group
def test_draft_group_not_annotated_without_spec_decode():
# The marker alone must not flag anything; the eagle semantics only apply
# when a speculative method is actually enabled.
config = _spec_decode_grouping_config()
config.speculative_config = None
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
assert not any(g.is_eagle_group for g in groups)
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
# No group carries the draft marker, so consumers fall back to
# conservative behavior that silently breaks reuse for Mamba groups.
# That must at least be visible.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
)
assert not any(g.is_eagle_group for g in groups)
assert "no KV cache group could be identified as the draft model's" in (
caplog_vllm.text
)
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
# Pure-attention models degrade gracefully under the consumers'
# conservative fallback (a one-block hit drop at most), so the warning
# stays silent to avoid noise on every unannotated EAGLE deployment.
specs = {
"target.attn.0": new_mla_spec(block_size=64),
"target.attn.1": new_mla_spec(block_size=64),
}
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" not in caplog_vllm.text
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
get_kv_cache_groups(
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
)
assert "could be identified as the draft model's" not in caplog_vllm.text
def _deepseek_v4_specs(model_version="deepseek_v4"):
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
at differing page sizes, with the MTP draft layer registered last."""
return {
"model.layers.0.self_attn.attn": new_mla_spec(),
"model.layers.1.self_attn.attn": new_mla_spec(),
"model.layers.2.self_attn.attn": new_swa_mla_spec(
head_size=1024, model_version=model_version
),
# The MTP block registers last, and its sliding-window size differs, so
# it lands in a group of its own.
"model.layers.3.self_attn.attn": new_swa_mla_spec(
head_size=1024, sliding_window=256, model_version=model_version
),
}
_DRAFT_SPEC_SHAPES = {
"uniform": lambda: {
"layers.0": new_kv_cache_spec(),
"draft.0": _as_draft(new_kv_cache_spec()),
},
"hybrid": _hybrid_specs,
"draft_spans_groups": lambda: {
"layers.0": new_mla_spec(),
"layers.1": new_swa_mla_spec(head_size=1024),
"mtp.0": _as_draft(new_mla_spec()),
"mtp.1": _as_draft(new_swa_mla_spec(head_size=1024)),
},
}
@pytest.mark.parametrize(
("method", "model_type"),
[("mtp", "deepseek_v4"), ("dspark", "deepseek_v4"), ("dspark", "deepseek_v41")],
"make_specs", _DRAFT_SPEC_SHAPES.values(), ids=list(_DRAFT_SPEC_SHAPES)
)
def test_deepseek_v4_draft_group_annotated_on_packed_path(method, model_type):
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
# carries no draft marker and only the positional rule can find it. This
# pins the pre-existing behaviour that the unified annotator must preserve.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method=method, model_type=model_type),
_deepseek_v4_specs(model_version=None),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def test_trailing_layer_fallback_applies_to_any_mtp_model():
# The positional rule is sound for every MTP drafter, not just DeepseekV4:
# MTP blocks reuse the target's decoder layer (no spec marker) and always
# register after every target layer. The model_type must not gate it.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="other"),
_deepseek_v4_specs(),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
def _qwen3_5_hybrid_specs(with_mtp_layer: bool):
"""Qwen3.5-shaped hybrid: repeating [GDN x3, full-attn x1] blocks, with
the MTP drafter's full-attn layer (spec-identical to the target's)
registered last."""
specs = {}
idx = 0
for _ in range(2):
for _ in range(3):
specs[f"model.layers.{idx}.linear_attn"] = new_mamba_spec(
mamba_cache_mode="align"
)
idx += 1
specs[f"model.layers.{idx}.self_attn.attn"] = new_kv_cache_spec()
idx += 1
if with_mtp_layer:
specs["mtp.layers.0.self_attn.attn"] = new_kv_cache_spec()
return specs
def test_qwen3_5_mtp_draft_group_annotated_on_hybrid_path(caplog_vllm):
# A hybrid mamba + full-attention model with an MTP drafter that is
# spec-indistinguishable from the target reaches the general multi-group
# path. The trailing-layer rule must locate the draft group there so the
# Mamba groups are not swept up by the flag-all consumer fallback.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method="mtp", model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
flagged = [g for g in groups if g.is_eagle_group]
assert len(flagged) == 1
assert "mtp.layers.0.self_attn.attn" in flagged[0].layer_names
for group in groups:
if any(
isinstance(spec, MambaSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
):
assert not group.is_eagle_group
assert "could be identified as the draft model's" not in caplog_vllm.text
@pytest.mark.parametrize("method", ["eagle", "eagle3", "dspark"])
def test_non_mtp_eagle_hybrid_still_warns(caplog_vllm, method):
# Other drafters are not covered by the trailing-layer rule, so an
# unidentifiable hybrid draft must still warn.
groups = get_kv_cache_groups(
_spec_decode_grouping_config(method=method, model_type="qwen3_5"),
_qwen3_5_hybrid_specs(with_mtp_layer=True),
)
assert not any(g.is_eagle_group for g in groups)
assert "could be identified as the draft model's" in caplog_vllm.text
def test_trailing_layer_fallback_requires_exact_partition():
# If the groups do not partition the layers exactly (e.g. a caller that
# dropped or duplicated layers), the positional rule is meaningless and
# must not fire.
from vllm.v1.core.kv_cache_planning import _annotate_eagle_groups
specs = _qwen3_5_hybrid_specs(with_mtp_layer=True)
config = _spec_decode_grouping_config(method="mtp", model_type="qwen3_5")
groups = get_kv_cache_groups(config, specs)
for g in groups:
g.is_eagle_group = False
# Remove one layer from its group: no longer an exact partition.
trimmed = [
KVCacheGroupSpec(
[n for n in g.layer_names if n != "model.layers.0.linear_attn"],
g.kv_cache_spec,
)
for g in groups
@pytest.mark.parametrize("pp", [1, 2])
def test_eagle_groups_are_the_groups_holding_draft_layers(make_specs, pp):
specs = make_specs()
draft_layers = {name for name, spec in specs.items() if spec.is_draft}
untagged = {name: replace(spec, is_draft=False) for name, spec in specs.items()}
groups = _spec_decode_kv_cache_groups([specs])
assert [(g.layer_names, g.kv_cache_spec) for g in groups] == [
(g.layer_names, g.kv_cache_spec)
for g in _spec_decode_kv_cache_groups([untagged])
]
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
assert not any(g.is_eagle_group for g in trimmed)
worker_specs = [
{n: s for n, s in specs.items() if n not in draft_layers},
{n: s for n, s in specs.items() if n in draft_layers},
]
scheduler_groups = _spec_decode_kv_cache_groups(
[specs] if pp == 1 else worker_specs
)
assert [g.is_eagle_group for g in scheduler_groups] == [
bool(draft_layers & set(g.layer_names)) for g in groups
]
def test_no_eagle_groups_without_eagle_block_drop():
groups = _spec_decode_kv_cache_groups([_hybrid_specs()], use_eagle_block_drop=False)
assert not any(g.is_eagle_group for g in groups)
@pytest.mark.parametrize(
"make_specs,warns",
[
(lambda: _hybrid_specs(draft=False), True),
(
lambda: {
"layers.0": new_kv_cache_spec(),
"draft.0": new_kv_cache_spec(),
},
False,
),
],
ids=["untagged_hybrid", "untagged_uniform_attention"],
)
def test_warns_only_when_untagged_draft_meets_mamba(make_specs, warns, caplog_vllm):
_spec_decode_kv_cache_groups([make_specs()])
warned = "could be identified as the draft model's" in caplog_vllm.text
assert warned == warns
_GQA_SPEC = FullAttentionSpec(
+16
View File
@@ -1303,6 +1303,22 @@ def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
@pytest.mark.parametrize(
"target",
[
new_kv_cache_spec(),
new_mla_spec(),
SinkFullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32, sink_len=4
),
],
)
def test_draft_layer_tag_is_not_part_of_the_cache_format(target):
draft = replace(target, is_draft=True)
assert draft == target and hash(draft) == hash(target)
assert type(target).merge([target, draft]) == target
@pytest.mark.parametrize(
"layer_type,dcp_size,expected_width",
[
@@ -929,14 +929,8 @@ class TestPushWriterNotifs:
notif_msg = f"{request_id}:1".encode()
w._pending_completion_notifs.put(notif_msg)
# transfer_topo is consulted only for the producer-side path; we
# make it a MagicMock because the D-side branch returns early.
w.transfer_topo = MagicMock()
w._get_new_notifs()
notified = w._get_new_notifs()
# Notif consumed; D-side just touches _recving_transfers.
assert notified == set()
assert request_id in w._recving_transfers
def test_get_transfer_results_evicts_completed_state(self):
@@ -1352,12 +1346,10 @@ class TestPushWriterNegative:
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
)
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
# Forward a completion notif for an unknown request_id.
w._pending_completion_notifs.put(b"never-heard-of-you:1")
notified = w._get_new_notifs()
assert notified == set()
w._get_new_notifs()
# Did not register anywhere.
assert "never-heard-of-you" not in w._recving_transfers
assert w.xfer_stats.data["num_notifications_after_expiry"] == [1]
@@ -1381,7 +1373,6 @@ class TestPushWriterNegative:
leases of tracked P-side requests on the engine main thread, and
ignore request IDs that aren't being tracked."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
# _handle_heartbeat reads ``self._lease_extension`` (set in the
# real ``__init__``).
w._lease_extension = 10
@@ -1395,8 +1386,7 @@ class TestPushWriterNegative:
# and another tracked one.
w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b")
notified = w._get_new_notifs()
assert notified == set()
w._get_new_notifs()
# Tracked leases were renewed strictly forward in time.
now = time.perf_counter()
@@ -1421,20 +1411,19 @@ class TestPushPipelineParallel:
"""Each PP stage WRITEs its own layers and sends one notif; D must
collect pp_size notifs before reporting the recv done."""
w = _StubWriterWorker.fresh()
w.transfer_topo = MagicMock()
request_id = "req-pp-2"
w._recving_metadata[request_id] = MagicMock(pp_size=2)
notif = f"{request_id}:1".encode()
# First stage: counted, not yet done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
w._get_new_notifs()
assert request_id not in w._recving_transfers
assert w.consumer_notification_counts_by_req[request_id] == 1
# Second (final) stage: now reported done.
w._pending_completion_notifs.put(notif)
assert w._get_new_notifs() == set()
w._get_new_notifs()
assert request_id in w._recving_transfers
assert request_id not in w.consumer_notification_counts_by_req
+2 -2
View File
@@ -110,8 +110,8 @@ def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
sampler = topk_topp_sampler.TopKTopPSampler()
logits = torch.randn(2, 8)
k = torch.full((2,), 2, dtype=torch.int32)
logits = torch.randn(2, 8, device=DEVICE_TYPE)
k = torch.full((2,), 2, dtype=torch.int32, device=DEVICE_TYPE)
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
token_ids, logits_to_return = sampler(logits, generators, k, None)
+3 -8
View File
@@ -103,15 +103,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
target_attn_layers = {"target_attn_1": mock.MagicMock()}
all_attn_layers = {**target_attn_layers, "draft_attn_1": mock.MagicMock()}
target_indexer_layers: dict = {}
all_indexer_layers: dict = {}
mock_get_layers.side_effect = [
target_attn_layers,
target_indexer_layers,
all_attn_layers,
all_indexer_layers,
]
mock_get_layers.side_effect = [target_attn_layers, all_attn_layers]
mock_pp_group = mock.MagicMock()
mock_pp_group.world_size = 1
@@ -138,6 +131,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
assert proposer.model.lm_head == target_model.lm_head
# MTP shares embed_tokens with target model
assert proposer.model.model.embed_tokens == target_model.model.embed_tokens
assert proposer._draft_attn_layer_names == {"draft_attn_1"}
assert all_attn_layers["draft_attn_1"].is_draft_layer is True
@pytest.mark.parametrize("num_speculative_tokens", [1])
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.engine.protocol import StreamingInput
from vllm.exceptions import VLLMValidationError
from vllm.outputs import STREAM_FINISHED, RequestOutput
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
@@ -107,6 +108,25 @@ def make_output(request_id: str, finished: bool) -> RequestOutput:
)
@pytest.mark.parametrize(
("is_mm_encoder_only", "is_diffusion"),
[(True, False), (False, True)],
ids=["encoder-only", "diffusion"],
)
def test_resumable_requests_rejected_for_non_decoder_models(
is_mm_encoder_only: bool, is_diffusion: bool
):
llm = MagicMock()
llm.vllm_config.is_mm_encoder_only = is_mm_encoder_only
llm.model_config.is_diffusion = is_diffusion
with pytest.raises(
VLLMValidationError,
match="Resumable requests are not supported",
):
AsyncLLM._validate_resumable_request(llm)
@pytest.mark.asyncio
async def test_generate_with_async_generator():
"""Test generate with an async input generator.
@@ -180,6 +200,8 @@ async def test_empty_input_stream_finishes_without_engine_request():
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
llm.vllm_config = MagicMock(is_mm_encoder_only=False)
llm.model_config = MagicMock(is_diffusion=False)
final_req = MagicMock()
final_req.request_id = "request-int"
@@ -204,6 +226,7 @@ async def test_empty_input_stream_finishes_without_engine_request():
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm._validate_resumable_request = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
@@ -9,6 +9,7 @@ import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from tests.v1.core.utils import create_scheduler as create_async_scheduler
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
from vllm.config import DeviceConfig, VllmConfig
from vllm.multimodal.inputs import (
@@ -179,6 +180,7 @@ class TestStreamingScheduler(unittest.TestCase):
request_id="session",
prompt_token_ids=[1, 2, 3],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
session.max_tokens = 2
@@ -195,6 +197,7 @@ class TestStreamingScheduler(unittest.TestCase):
assert session.sampling_params.max_tokens == 10
assert session.max_tokens == 10
session.append_output_token_ids(11)
session.num_computed_tokens = len(session.prompt_token_ids)
new_request2 = DummyRequest(
@@ -216,6 +219,7 @@ class TestStreamingScheduler(unittest.TestCase):
request_id="session",
prompt_token_ids=[1, 2, 3],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
new_request = DummyRequest(
@@ -246,6 +250,7 @@ class TestStreamingScheduler(unittest.TestCase):
prompt_token_ids=[1, 2, 3],
mm_features=[mm_feature],
)
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
mm_feature = MultiModalFeatureSpec(
@@ -316,6 +321,7 @@ class TestStreamingScheduler(unittest.TestCase):
)
scheduler.add_request(session)
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
session.append_output_token_ids(10)
session.num_computed_tokens = len(session.prompt_token_ids)
next_request = DummyRequest(
@@ -716,3 +722,54 @@ def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
assert output.num_scheduled_tokens["session"] == 2
@pytest.mark.parametrize("chunk_queued_before_stop", [False, True])
def test_session_resumes_after_stop_while_preempted(chunk_queued_before_stop):
"""A session preempted with a step in flight can stop when that step's
output is delivered. It must stay schedulable and keep its tokens."""
scheduler = create_async_scheduler(async_scheduling=True)
session = DummyRequest("session", prompt_token_ids=list(range(10)))
scheduler.add_request(session)
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
in_flight = scheduler.schedule()
next_chunk = DummyRequest("session", prompt_token_ids=[20, 21])
if chunk_queued_before_stop:
scheduler.add_request(next_chunk)
scheduler.running.remove(session)
scheduler._preempt_request(session, timestamp=0.0)
scheduler.update_from_output(in_flight, _model_output("session", [STOP_TOKEN]))
# The parked variant holds a streaming-input slot until the next chunk.
assert scheduler.num_waiting_for_streaming_input == (
0 if chunk_queued_before_stop else 1
)
if not chunk_queued_before_stop:
scheduler.add_request(next_chunk)
output = scheduler.schedule()
assert scheduler.num_waiting_for_streaming_input == 0
expected = list(range(10)) + [10, 20, 21]
assert list(session.all_token_ids) == expected
assert list(session.prompt_token_ids) == expected
assert output.num_scheduled_tokens["session"] == len(expected)
def test_non_resumable_request_stays_dequeued_after_stop_while_preempted():
"""Only resumable requests continue after a stop; a request that finishes
while preempted must not be put back in a queue."""
scheduler = create_async_scheduler(async_scheduling=True)
request = DummyRequest("req", resumable=False, prompt_token_ids=list(range(10)))
scheduler.add_request(request)
scheduler.update_from_output(scheduler.schedule(), _model_output("req", [10]))
in_flight = scheduler.schedule()
scheduler.running.remove(request)
scheduler._preempt_request(request, timestamp=0.0)
scheduler.update_from_output(in_flight, _model_output("req", [STOP_TOKEN]))
assert request.status == RequestStatus.FINISHED_STOPPED
assert request not in scheduler.waiting
assert request not in scheduler.kv_holding_waiting
assert request not in scheduler.deferred_waiting
assert "req" not in scheduler.schedule().num_scheduled_tokens
+24
View File
@@ -92,6 +92,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
get_attn_backend=lambda backend=backend: backend,
is_draft_layer=False,
)
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
monkeypatch.setattr(
@@ -112,6 +113,29 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
assert all(spec.block_size == block_size for spec in specs.values())
def test_get_kv_cache_spec_tags_draft_layers(monkeypatch):
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
)
backend = SimpleNamespace(
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda _=None: [16],
)
layers = {
name: SimpleNamespace(
get_kv_cache_spec=lambda _: spec,
get_attn_backend=lambda: backend,
is_draft_layer=name == "draft",
)
for name in ("target", "draft")
}
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
config = SimpleNamespace(attention_config=SimpleNamespace(hisparse_config=None))
specs = attn_utils.get_kv_cache_spec(config)
assert specs["draft"].is_draft and not specs["target"].is_draft
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
self.support = support
+101
View File
@@ -305,3 +305,104 @@ def test_dummy_request_slot_mapping_is_pad():
num_tokens_padded=3,
)
assert dummy[0].tolist() == [PAD_SLOT_ID] * 3
@pytest.fixture
def populated_block_tables():
tables = BlockTables([16, 32], 8, 64, [3, 5], torch.device("cuda"))
for req_index in range(8):
tables.append_block_ids(
req_index, ([req_index + 1] * 3, [req_index + 1] * 5), overwrite=True
)
tables.apply_staged_writes()
tables.gather_block_tables(torch.arange(8, device="cuda"), num_reqs_padded=8)
return tables
@pytest.mark.parametrize("previous_dummy", [False, True])
@pytest.mark.parametrize("next_dummy", [False, True])
def test_shared_block_tables_clear_rows_from_larger_batch(
populated_block_tables, previous_dummy, next_dummy
):
"""A smaller target batch must leave no stale rows for larger draft graphs."""
tables = populated_block_tables
if previous_dummy:
for table in tables.get_dummy_block_tables(8):
table.fill_(99) # Profiling supplies valid context/state slots.
ptrs = [table.data_ptr() for table in tables.input_block_tables]
idx = torch.tensor([7, 2, 0, 5, 1], device="cuda")
for num_reqs in (3, 5, 0):
if next_dummy:
out = tables.get_dummy_block_tables(num_reqs)
num_rows = num_reqs
else:
num_rows = num_reqs + 1
out = tables.gather_block_tables(idx[:num_reqs], num_rows)
for table, view, ptr in zip(tables.input_block_tables, out, ptrs):
assert view.shape == (num_rows, table.shape[1])
assert table.data_ptr() == ptr
if num_rows:
assert view.data_ptr() == ptr
assert torch.count_nonzero(table[num_reqs:]).item() == 0
if next_dummy:
assert torch.count_nonzero(view).item() == 0
view.fill_(99)
else:
expected = (idx[:num_reqs, None] + 1).expand(-1, table.shape[1])
torch.testing.assert_close(view[:num_reqs], expected.to(torch.int32))
def test_external_block_table_output_preserves_shared_rows(populated_block_tables):
"""PCP-local gathers must neither clear shared rows nor reset their tracking."""
tables = populated_block_tables
previous = [table.clone() for table in tables.input_block_tables]
out = tuple(torch.full_like(table[:3], -1) for table in previous)
out_ptrs = torch.tensor(
[table.data_ptr() for table in out], dtype=torch.uint64, device="cuda"
)
idx = torch.tensor([7, 2], device="cuda")
result = tables.gather_block_tables(idx, 3, out=out, out_ptrs=out_ptrs)
for table, before, view in zip(tables.input_block_tables, previous, result):
torch.testing.assert_close(table, before)
assert view[:, 0].tolist() == [8, 3, 0]
tables.gather_block_tables(idx[:1], num_reqs_padded=1)
for table in tables.input_block_tables:
assert torch.count_nonzero(table[1:]).item() == 0
@pytest.mark.parametrize("use_cuda_graph", [False, True])
def test_larger_draft_batch_uses_null_kpool_slots(
populated_block_tables, use_cuda_graph
):
"""Draft padding must not resolve to another request's physical KPool slots."""
from vllm.v1.attention.backends.mla.indexer import compute_kpool_tail_slot_mapping
tables = populated_block_tables
idx = torch.tensor([2, 0, 4, 1, 3], device="cuda")
query_start_loc = torch.arange(9, dtype=torch.int32, device="cuda").clamp_max_(5)
slot_mapping = torch.full((8,), -1, dtype=torch.int64, device="cuda")
positions = torch.arange(8, dtype=torch.int64, device="cuda")
def compute_slots():
return compute_kpool_tail_slot_mapping(
slot_mapping,
tables.input_block_tables[0],
query_start_loc,
positions,
num_actual_tokens=8,
num_reqs=8,
kpool=8,
)
if use_cuda_graph:
compute_slots()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
slots = compute_slots()
tables.gather_block_tables(idx, num_reqs_padded=5)
if use_cuda_graph:
graph.replay()
else:
slots = compute_slots()
assert slots.tolist() == [24, 9, 42, 19, 36, -1, -1, -1]
+25 -3
View File
@@ -22,7 +22,12 @@ def _make_connector(
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_transfer_results.return_value = KVConnectorTransferResults()
def get_transfer_results(_finished_req_ids):
events.append("results")
return KVConnectorTransferResults()
backend.get_transfer_results.side_effect = get_transfer_results
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
@@ -73,7 +78,7 @@ def test_load_start_phase(
)
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
connector.no_forward(_scheduler_output(False))
assert events == ["handle", "bind", "start", "wait", "clear"]
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
def test_deferred_finalize_collects_results_before_saving(
monkeypatch: pytest.MonkeyPatch,
):
"""With a drafter, the transfer results are collected right after the
target forward, and the saves and metadata are finalized after the
drafter writes its KV."""
events: list[str] = []
connector = _make_connector(monkeypatch, events)
connector.pre_forward(_scheduler_output(False))
connector.post_forward(set(), defer_finalize=True)
assert events == ["handle", "bind", "start", "results"]
connector.finalize_step()
assert events == ["handle", "bind", "start", "results", "wait", "clear"]
@@ -88,6 +88,7 @@ def _make_runner(
"""Stub model runner exposing only what the warmup entry points read."""
return SimpleNamespace(
num_speculative_steps=num_spec_steps,
speculative_config=SimpleNamespace() if num_spec_steps else None,
adaptive_verification=None,
rejection_sampler=None,
decode_query_len=num_spec_steps + 1,
+25
View File
@@ -74,6 +74,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
runner = SimpleNamespace(
adaptive_verification=manager,
rejection_sampler=rejection_sampler,
speculative_config=SimpleNamespace(),
)
def run_steps(model_runner, execute, sample, greedy=False):
@@ -91,3 +92,27 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
assert runner.adaptive_verification is manager
assert manager.cost_tables is None
assert rejection_sampler.enable_adaptive_verification
@pytest.mark.parametrize("speculative", [False, True])
def test_kernel_warmup_runs_same_rounds_on_every_pp_stage(monkeypatch, speculative):
"""PP receivers need a matching sender for every speculative warmup round."""
rounds = []
monkeypatch.setattr(
warmup,
"_warmup_kernels",
lambda runner, execute, sample, greedy=False: rounds.append(greedy),
)
for is_last_pp_rank in (False, True):
rounds.clear()
runner = SimpleNamespace(
adaptive_verification=None,
speculative_config=SimpleNamespace() if speculative else None,
rejection_sampler=(
SimpleNamespace(enable_adaptive_verification=False)
if is_last_pp_rank and speculative
else None
),
)
warmup.warmup_kernels(runner, _fail, _fail)
assert rounds == ([False, True] if speculative else [False])