Merge branch 'main' into patches
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
@@ -10,6 +11,7 @@ import pytest
|
||||
|
||||
from vllm.benchmarks.forward.batch import build_batch
|
||||
from vllm.benchmarks.forward.main import run_case
|
||||
from vllm.benchmarks.forward.profile import summarize
|
||||
from vllm.benchmarks.forward.shapes import jitter_kv_lens, parse_case
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -97,6 +99,35 @@ def test_bench_forward_rejects_oversized_verify(capacity):
|
||||
run_case(runner, case, SimpleNamespace())
|
||||
|
||||
|
||||
def test_profile_summary():
|
||||
"""Each replay's leading restore/rendezvous activities are left out, and
|
||||
kernels are grouped by name without template or function arguments."""
|
||||
|
||||
def act(name, ts, dur, cat="kernel"):
|
||||
return {"cat": cat, "name": name, "ts": ts, "dur": dur}
|
||||
|
||||
replay = [
|
||||
("Memcpy DtoD (Device -> Device)", "gpu_memcpy", 5),
|
||||
("void ns::gemm<128>(Params)", "kernel", 10),
|
||||
("void ns::gemm<64>(Params)", "kernel", 20),
|
||||
("std::enable_if<(1>0), void>::type ns::norm<float>(float*)", "kernel", 4),
|
||||
]
|
||||
events = [{"cat": "cpu_op", "name": "x", "ts": 0, "dur": 1}]
|
||||
for r in range(2):
|
||||
for i, (name, cat, dur) in enumerate(replay):
|
||||
events.append(act(name, 1000 * r + 100 * i, dur, cat))
|
||||
summary = summarize(events, replays=2, skip=1)
|
||||
assert summary["pre_forward_us"] == 5
|
||||
assert summary["busy_us"] == 34
|
||||
assert summary["span_us"] == 204
|
||||
assert [(k["name"], k["us"], k["calls"]) for k in summary["kernels"]] == [
|
||||
("ns::gemm", 30, 2),
|
||||
("ns::norm", 4, 1),
|
||||
]
|
||||
with pytest.raises(ValueError, match="replays"):
|
||||
summarize(events[:-1], replays=2, skip=1)
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
# TODO(woosuk): Support ROCm and re-enable this test there.
|
||||
@pytest.mark.skipif(
|
||||
@@ -126,6 +157,8 @@ def test_bench_forward(tmp_path):
|
||||
"2",
|
||||
"--output-json",
|
||||
str(output),
|
||||
"--profile-dir",
|
||||
str(tmp_path / "profile"),
|
||||
]
|
||||
result = subprocess.run(command, capture_output=True, text=True)
|
||||
print(result.stdout)
|
||||
@@ -139,3 +172,7 @@ def test_bench_forward(tmp_path):
|
||||
assert len(rank0["batch_order"]) == len(row["requests"])
|
||||
assert rank0["finite"] and rank0["graph_deterministic"]
|
||||
assert row["median_us"] > 0
|
||||
profile = rank0["profile"]
|
||||
assert profile["top_kernels"] and profile["span_us"] > 0
|
||||
summary = json.loads(Path(profile["summary"]).read_text())
|
||||
assert Path(summary["trace"]).exists()
|
||||
|
||||
@@ -462,8 +462,8 @@ def test_flashinfer_standalone_workspace_size(
|
||||
@pytest.mark.parametrize(
|
||||
("node_count", "is_blackwell", "expected"),
|
||||
[
|
||||
(1, False, ("trtllm", False)),
|
||||
(1, True, ("trtllm", False)),
|
||||
(1, False, ("mnnvl", True)),
|
||||
(1, True, ("mnnvl", True)),
|
||||
(2, False, (None, False)),
|
||||
(2, True, ("mnnvl", False)),
|
||||
],
|
||||
|
||||
@@ -12,8 +12,10 @@ import pytest_asyncio
|
||||
from tests.utils import RemoteLaunchRenderServer
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
|
||||
GenerateTokensResponse,
|
||||
GenerateTokensStreamResponse,
|
||||
)
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
@@ -1579,3 +1581,43 @@ def test_derender_chat_keeps_parser_tool_call_id():
|
||||
|
||||
(tool_call,) = choice.message.tool_calls
|
||||
assert tool_call.id == "functions.get_weather:1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("return_token_ids", [None, False, True])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_derender_completion_preserves_requested_token_ids(
|
||||
tokenizer, return_token_ids, stream
|
||||
):
|
||||
renderer = MagicMock()
|
||||
renderer.get_tokenizer.return_value = tokenizer
|
||||
renderer._executor = None
|
||||
model_config = MagicMock()
|
||||
model_config.model = MODEL_NAME
|
||||
model_config.hf_config.model_type = "llama"
|
||||
derenderer = OnlineDerenderer(
|
||||
model_config,
|
||||
renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="string",
|
||||
)
|
||||
token_ids = tokenizer.encode("Hello world", add_special_tokens=False)
|
||||
request = CompletionRequest(
|
||||
model=MODEL_NAME, prompt="Hi", return_token_ids=return_token_ids
|
||||
)
|
||||
payload = _make_generate_response(token_ids, finish_reason="length")
|
||||
if stream:
|
||||
response, _ = await derenderer.derender_completion_stream(
|
||||
MODEL_NAME,
|
||||
GenerateTokensStreamResponse.model_validate(payload),
|
||||
completion_request=request,
|
||||
)
|
||||
choices = response.choices
|
||||
else:
|
||||
choices, _, _ = await derenderer.derender_completion(
|
||||
[GenerateTokensResponse.model_validate(payload)],
|
||||
completion_request=request,
|
||||
)
|
||||
assert choices[0].text == tokenizer.decode(token_ids)
|
||||
assert choices[0].token_ids == (token_ids if return_token_ids else None)
|
||||
|
||||
@@ -18,13 +18,16 @@ from vllm.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
## Parallel Processes Utils
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
LAUNCH_TIMEOUT_S = 900
|
||||
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
|
||||
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
|
||||
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
|
||||
@@ -147,26 +147,15 @@ def test_fp4_env_defaults():
|
||||
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"use_aiter",
|
||||
"use_fp4_asm_gemm",
|
||||
"use_fp4bmm",
|
||||
),
|
||||
[
|
||||
(True, True, True),
|
||||
(True, True, False),
|
||||
(True, False, True),
|
||||
(False, True, True),
|
||||
],
|
||||
("use_aiter", "use_fp4bmm"),
|
||||
[(True, True), (True, False), (False, True)],
|
||||
)
|
||||
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
use_aiter,
|
||||
use_fp4_asm_gemm,
|
||||
use_fp4bmm,
|
||||
monkeypatch,
|
||||
):
|
||||
@@ -182,22 +171,16 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
_assert_aiter_supported()
|
||||
|
||||
on_gfx950_value = on_gfx950()
|
||||
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
|
||||
expected_asm_gemm = use_aiter and on_gfx950_value
|
||||
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv(
|
||||
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
|
||||
"1" if use_fp4_asm_gemm else "0",
|
||||
)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert (
|
||||
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
|
||||
)
|
||||
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
|
||||
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
|
||||
|
||||
|
||||
@@ -695,24 +678,52 @@ def test_asm_fp4_scale_swizzle_rejects_non_2d():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
|
||||
"""A misaligned weight_scale must disable ASM and take the Triton path."""
|
||||
@pytest.mark.parametrize(
|
||||
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
|
||||
[
|
||||
(True, True, (1280, 4096), (1280, 256), True),
|
||||
(True, False, (1280, 4096), (1280, 256), False),
|
||||
(False, True, (1280, 4096), (1280, 256), False),
|
||||
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
|
||||
(True, True, (2560, 160), (2560, 10), False),
|
||||
],
|
||||
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
|
||||
)
|
||||
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
|
||||
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
|
||||
):
|
||||
"""A layer takes the ASM layout only when aiter has tuned configs for both
|
||||
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
|
||||
Otherwise it stays on the Triton layout."""
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
|
||||
lambda n, k: preshuffle_tuned,
|
||||
)
|
||||
kernel = object.__new__(AiterMxfp4LinearKernel)
|
||||
kernel.use_asm_gemm = True
|
||||
kernel.asm_gemm_supported = True
|
||||
kernel.use_asm_gemm = False
|
||||
kernel.out_dtype = torch.bfloat16
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = Parameter(
|
||||
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = Parameter(
|
||||
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
|
||||
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.use_asm_gemm is False
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == (10, 2560)
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
assert kernel.use_asm_gemm is expected
|
||||
if not expected:
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HEAD_DIM = 64
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() or not torch.cuda.is_available(),
|
||||
reason="Muse Spark kernel requires a CUDA or ROCm GPU",
|
||||
)
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos, rotary_dim, base, device, dtype=torch.float32):
|
||||
"""The table ``RotaryEmbedding`` builds: cos||sin, fp32 for Muse-Spark."""
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (
|
||||
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
|
||||
/ rotary_dim
|
||||
)
|
||||
)
|
||||
t = torch.arange(max_pos, dtype=torch.float32, device=device)
|
||||
freqs = torch.einsum("i,j->ij", t, inv_freq)
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1).to(dtype)
|
||||
|
||||
|
||||
def qk_norm_rope_ref(x, positions, cos_sin_cache, rotary_dim, eps):
|
||||
"""fp32 reference for one q or k branch, rounded once; no table means NoPE."""
|
||||
out = x.float()
|
||||
out = out * torch.rsqrt(out.pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
if cos_sin_cache is not None:
|
||||
half = rotary_dim // 2
|
||||
cs = cos_sin_cache[positions].float()
|
||||
cos = cs[..., :half].unsqueeze(1)
|
||||
sin = cs[..., half:].unsqueeze(1)
|
||||
# GPT-J interleaved: the rotated pair is (2i, 2i+1).
|
||||
x0 = out[..., :rotary_dim:2]
|
||||
x1 = out[..., 1:rotary_dim:2]
|
||||
out = out.clone()
|
||||
out[..., :rotary_dim:2] = x0 * cos - x1 * sin
|
||||
out[..., 1:rotary_dim:2] = x0 * sin + x1 * cos
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
def assert_fp8_cache_close(got, expected, exact_fraction=0.999):
|
||||
"""Compare raw cache codes in the platform's FP8 encoding."""
|
||||
diff = (got.int() - expected.int()).abs()
|
||||
within = diff <= 1
|
||||
assert bool(within.all()), (
|
||||
f"fp8 cache differs by more than one code in {int((~within).sum())} "
|
||||
f"of {diff.numel()} elements"
|
||||
)
|
||||
exact = float((diff == 0).float().mean())
|
||||
assert exact >= exact_fraction, (
|
||||
f"only {exact:.5f} of the fp8 cache is bit-identical to "
|
||||
f"reshape_and_cache_flash, expected at least {exact_fraction}"
|
||||
)
|
||||
|
||||
|
||||
def make_case(num_tokens, nq, nkv, dtype, device, with_gate=True, seed=0):
|
||||
torch.manual_seed(seed)
|
||||
qsz, kvsz = nq * HEAD_DIM, nkv * HEAD_DIM
|
||||
gate = qsz if with_gate else 0
|
||||
qkv = torch.randn(num_tokens, qsz + 2 * kvsz + gate, dtype=dtype, device=device)
|
||||
return qkv, qsz, kvsz, gate
|
||||
|
||||
|
||||
def split_qkv(qkv, qsz, kvsz, gate):
|
||||
return qkv.split([qsz, kvsz, kvsz, gate], dim=-1)
|
||||
|
||||
|
||||
def check_qk_norm_rope(
|
||||
num_tokens,
|
||||
nq,
|
||||
nkv,
|
||||
dtype,
|
||||
use_rope,
|
||||
rotary_dim=HEAD_DIM,
|
||||
with_gate=True,
|
||||
rope_dtype=torch.float32,
|
||||
):
|
||||
device, eps, max_pos = "cuda", 1e-5, 4096
|
||||
|
||||
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, with_gate)
|
||||
qkv_orig = qkv.clone()
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device, rope_dtype)
|
||||
if use_rope
|
||||
else None
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
HEAD_DIM,
|
||||
eps,
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim if use_rope else 0,
|
||||
)
|
||||
|
||||
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
|
||||
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
|
||||
|
||||
q_ref = qk_norm_rope_ref(
|
||||
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = qk_norm_rope_ref(
|
||||
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, kvsz)
|
||||
|
||||
# rsqrtf and lane summation order cost a couple of fp32 ulp, well inside
|
||||
# one ulp of the stored dtype.
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_rope", [True, False])
|
||||
def test_qk_norm_rope(dtype, use_rope):
|
||||
check_qk_norm_rope(17, 32, 4, dtype, use_rope)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,variant",
|
||||
[
|
||||
pytest.param(64, {"rotary_dim": 32}, id="partial_rotary_dim"),
|
||||
# A [q | k | v] row (no o-gate) is addressed by its own stride.
|
||||
pytest.param(33, {"with_gate": False}, id="no_gate_row"),
|
||||
pytest.param(128, {"rope_dtype": torch.bfloat16}, id="bf16_cos_sin_cache"),
|
||||
],
|
||||
)
|
||||
def test_qk_norm_rope_layout_variants(num_tokens, variant):
|
||||
check_qk_norm_rope(num_tokens, 32, 4, torch.bfloat16, True, **variant)
|
||||
|
||||
|
||||
# Cover narrow/wide head groups for TP4/TP8 and the one-chunk-per-token case.
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,nq,nkv,kv_cache_dtype,use_rope",
|
||||
[
|
||||
pytest.param(1, 32, 4, "auto", True, id="tp4-decode"),
|
||||
pytest.param(1, 16, 2, "fp8", False, id="tp8-decode-fp8-nope"),
|
||||
pytest.param(1025, 32, 4, "auto", True, id="tp4-group2"),
|
||||
pytest.param(1025, 32, 4, "fp8", True, id="tp4-group2-fp8"),
|
||||
pytest.param(1537, 32, 4, "auto", True, id="tp4-group4"),
|
||||
pytest.param(1537, 32, 4, "auto", False, id="tp4-nope"),
|
||||
pytest.param(3073, 16, 2, "auto", True, id="tp8-group4"),
|
||||
pytest.param(3073, 16, 2, "fp8", False, id="tp8-fp8-nope"),
|
||||
pytest.param(40000, 2, 1, "auto", True, id="single-chunk-per-token"),
|
||||
],
|
||||
)
|
||||
def test_kv_insert(num_tokens, nq, nkv, kv_cache_dtype, use_rope):
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-5
|
||||
rotary_dim, max_pos, block_size = HEAD_DIM, 4096, 16
|
||||
|
||||
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, seed=1)
|
||||
qkv_orig = qkv.clone()
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device) if use_rope else None
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
storage = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, nkv, block_size, 2 * HEAD_DIM, dtype=storage, device=device
|
||||
)
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
if num_tokens > 1:
|
||||
# -1 marks a token the scheduler did not place; the kernel must skip it
|
||||
# and leave the cache alone.
|
||||
slot_mapping[::3] = -1
|
||||
|
||||
# Non-unit scales, so a kernel that ignored them would fail loudly.
|
||||
k_scale = torch.tensor(0.37, dtype=torch.float32, device=device)
|
||||
v_scale = torch.tensor(1.83, dtype=torch.float32, device=device)
|
||||
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
HEAD_DIM,
|
||||
eps,
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim if use_rope else 0,
|
||||
slot_mapping=slot_mapping,
|
||||
kv_cache=kv_cache,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_scale=k_scale if kv_cache_dtype != "auto" else None,
|
||||
v_scale=v_scale if kv_cache_dtype != "auto" else None,
|
||||
)
|
||||
|
||||
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
|
||||
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
|
||||
|
||||
q_ref = qk_norm_rope_ref(
|
||||
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = qk_norm_rope_ref(
|
||||
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, kvsz)
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
|
||||
|
||||
expected = torch.zeros_like(kv_cache)
|
||||
exp_k, exp_v = expected.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
ops.reshape_and_cache_flash(
|
||||
k_out.view(num_tokens, nkv, HEAD_DIM),
|
||||
v_out.view(num_tokens, nkv, HEAD_DIM),
|
||||
exp_k,
|
||||
exp_v,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
if kv_cache_dtype == "auto":
|
||||
torch.testing.assert_close(kv_cache, expected, rtol=0, atol=0)
|
||||
else:
|
||||
assert_fp8_cache_close(kv_cache, expected)
|
||||
|
||||
|
||||
# The fused Q output requires E4M3FN; scaled_fp8_quant uses the platform dtype.
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm() and current_platform.fp8_dtype() != torch.float8_e4m3fn,
|
||||
reason="Fused E4M3FN Q output cannot be compared with ROCm FNUZ quantization",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"head_dim,num_tokens,rotary_fraction,use_cuda_graph",
|
||||
[
|
||||
pytest.param(64, 17, 0, False, id="head64-nope"),
|
||||
pytest.param(128, 17, 0.5, False, id="head128-partial-rope"),
|
||||
pytest.param(256, 17, 0, False, id="head256-nope"),
|
||||
pytest.param(256, 3073, 0.5, False, id="head256-partial-rope-large"),
|
||||
pytest.param(64, 3073, 0.5, True, id="graph-rope"),
|
||||
pytest.param(64, 3073, 0, True, id="graph-nope"),
|
||||
],
|
||||
)
|
||||
def test_fused_query_quant_matches_separate_quant(
|
||||
head_dim, num_tokens, rotary_fraction, use_cuda_graph
|
||||
):
|
||||
"""Preserve the rounding boundary, scales, padded Q, and KV/gate writes."""
|
||||
torch.manual_seed(42)
|
||||
nq, nkv, block_size = 16, 2, 16
|
||||
qsz, kvsz = nq * head_dim, nkv * head_dim
|
||||
qkv = torch.randn(num_tokens, 2 * (qsz + kvsz), device="cuda", dtype=torch.bfloat16)
|
||||
original = qkv.clone()
|
||||
reference = qkv.clone()
|
||||
q_fp8 = torch.empty(num_tokens, qsz, device="cuda", dtype=torch.float8_e4m3fn)
|
||||
positions = torch.arange(num_tokens, device="cuda", dtype=torch.int64)
|
||||
rotary_dim = int(head_dim * rotary_fraction)
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(num_tokens, rotary_dim, 500_000.0, "cuda")
|
||||
if rotary_dim
|
||||
else None
|
||||
)
|
||||
cache = torch.zeros(
|
||||
(num_tokens + block_size - 1) // block_size,
|
||||
nkv,
|
||||
block_size,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
expected_cache = torch.zeros_like(cache)
|
||||
slots = positions.clone()
|
||||
slots[::3] = -1
|
||||
# Small enough that part of Q saturates the e4m3 range.
|
||||
scale = torch.tensor(0.001, device="cuda", dtype=torch.float32)
|
||||
kwargs = dict(
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim,
|
||||
slot_mapping=slots,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype="fp8",
|
||||
k_scale=torch.tensor(0.37, device="cuda"),
|
||||
v_scale=torch.tensor(1.83, device="cuda"),
|
||||
)
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
reference,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
1e-5,
|
||||
kv_cache=expected_cache,
|
||||
**kwargs,
|
||||
)
|
||||
expected_q, _ = ops.scaled_fp8_quant(reference[:, :qsz], scale=scale)
|
||||
|
||||
def run():
|
||||
qkv.copy_(original)
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
1e-5,
|
||||
kv_cache=cache,
|
||||
q_fp8_out=q_fp8,
|
||||
q_scale=scale,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
run()
|
||||
if use_cuda_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
qkv.zero_()
|
||||
cache.zero_()
|
||||
q_fp8.zero_()
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
q_fp8.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(cache, expected_cache, rtol=0, atol=0)
|
||||
torch.testing.assert_close(qkv[:, :qsz], original[:, :qsz], rtol=0, atol=0)
|
||||
torch.testing.assert_close(qkv[:, qsz:], reference[:, qsz:], rtol=0, atol=0)
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weighted RMSNorm + partial NeoX RoPE with separate paged K/V caches."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not torch.cuda.is_available(),
|
||||
reason="Step-5 kernel is only built for CUDA",
|
||||
)
|
||||
|
||||
|
||||
def norm_rope_reference(x, weight, cos_sin, positions, rotary_dim, eps, bias):
|
||||
xf = x.float()
|
||||
normed = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps)
|
||||
out = (normed * (weight.float() + bias)).to(x.dtype)
|
||||
half = rotary_dim // 2
|
||||
cos, sin = cos_sin[positions].float().unsqueeze(1).chunk(2, dim=-1)
|
||||
x0, x1 = out[..., :half].float(), out[..., half:rotary_dim].float()
|
||||
out[..., :half] = x0 * cos - x1 * sin
|
||||
out[..., half:rotary_dim] = x0 * sin + x1 * cos
|
||||
return out
|
||||
|
||||
|
||||
def check_norm_rope_and_cache(
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
head_dim=128,
|
||||
rotary_dim=64,
|
||||
num_tokens=19,
|
||||
block_size=16,
|
||||
fp32_weights=False,
|
||||
bias=0.0,
|
||||
strided_cache=False,
|
||||
use_cuda_graph=False,
|
||||
):
|
||||
"""Cover rounding, 192-wide RoPE, padded rows and short/negative slots."""
|
||||
torch.manual_seed(0)
|
||||
nq, nkv, max_pos, eps = 5, 2, 64, 1e-5
|
||||
width = (nq + 2 * nkv) * head_dim
|
||||
storage = torch.randn(num_tokens, width + 8, device="cuda", dtype=dtype)
|
||||
qkv = storage[:, :width]
|
||||
original = storage.clone()
|
||||
weight_dtype = torch.float32 if fp32_weights else dtype
|
||||
qw = torch.randn(head_dim, device="cuda", dtype=weight_dtype)
|
||||
kw = torch.randn_like(qw)
|
||||
angles = torch.randn(max_pos, rotary_dim // 2, device="cuda")
|
||||
rope_dtype = torch.float32 if fp32_weights else dtype
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(rope_dtype)
|
||||
positions = torch.randint(max_pos, (num_tokens,), device="cuda")
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
shape = (num_blocks, block_size, nkv, head_dim)
|
||||
if strided_cache:
|
||||
shape = (num_blocks, nkv, block_size, head_dim)
|
||||
key_cache = torch.zeros(shape, device="cuda", dtype=cache_dtype)
|
||||
value_cache = torch.zeros_like(key_cache)
|
||||
if strided_cache:
|
||||
key_cache = key_cache.transpose(1, 2)
|
||||
value_cache = value_cache.transpose(1, 2)
|
||||
# A short mapping models graph padding; negative slots must stay unwritten.
|
||||
num_slots = max(0, num_tokens - 3) if num_tokens > 1 else num_tokens
|
||||
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_slots]
|
||||
slots[1::3] = -1
|
||||
k_scale = torch.tensor(0.37, device="cuda")
|
||||
v_scale = torch.tensor(1.83, device="cuda")
|
||||
|
||||
def run():
|
||||
return ops.fused_qknorm_rope_cache(
|
||||
qkv,
|
||||
qw,
|
||||
kw,
|
||||
cos_sin,
|
||||
positions,
|
||||
slots,
|
||||
key_cache,
|
||||
value_cache,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
eps,
|
||||
norm_weight_bias=bias,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
q, k, v = run()
|
||||
if use_cuda_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
q, k, v = run()
|
||||
key_cache.zero_()
|
||||
value_cache.zero_()
|
||||
graph.replay()
|
||||
|
||||
q_in, k_in, v_in = qkv.split(
|
||||
(nq * head_dim, nkv * head_dim, nkv * head_dim), dim=-1
|
||||
)
|
||||
tol = 1e-2 if dtype == torch.bfloat16 else 2e-3
|
||||
for actual, src, weight, heads in ((q, q_in, qw, nq), (k, k_in, kw, nkv)):
|
||||
expected = norm_rope_reference(
|
||||
src.reshape(num_tokens, heads, head_dim),
|
||||
weight,
|
||||
cos_sin,
|
||||
positions,
|
||||
rotary_dim,
|
||||
eps,
|
||||
bias,
|
||||
).reshape_as(actual)
|
||||
torch.testing.assert_close(actual, expected, rtol=tol, atol=tol)
|
||||
torch.testing.assert_close(v, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(storage, original, rtol=0, atol=0)
|
||||
|
||||
expected_k, expected_v = torch.zeros_like(key_cache), torch.zeros_like(value_cache)
|
||||
if num_slots:
|
||||
ops.reshape_and_cache_flash(
|
||||
k.reshape(num_tokens, nkv, head_dim),
|
||||
v.reshape(num_tokens, nkv, head_dim),
|
||||
expected_k,
|
||||
expected_v,
|
||||
slots,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
for actual, expected in ((key_cache, expected_k), (value_cache, expected_v)):
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Reciprocal multiplication versus division may differ by one code.
|
||||
assert (actual.int() - expected.int()).abs().max() <= 1
|
||||
else:
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(dict(head_dim=64, rotary_dim=32, num_tokens=1), id="decode"),
|
||||
pytest.param(
|
||||
dict(block_size=7, fp32_weights=True, bias=1.0, strided_cache=True),
|
||||
id="strided-cache-fp32-weights",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=192, rotary_dim=128, fp32_weights=True),
|
||||
id="head192-partial-rope",
|
||||
),
|
||||
pytest.param(
|
||||
dict(
|
||||
head_dim=192,
|
||||
rotary_dim=192,
|
||||
block_size=7,
|
||||
bias=1.0,
|
||||
strided_cache=True,
|
||||
kv_cache_dtype="fp8",
|
||||
),
|
||||
id="head192-full-rope-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
dict(
|
||||
head_dim=256, rotary_dim=256, strided_cache=True, kv_cache_dtype="fp8"
|
||||
),
|
||||
id="head256-full-rope-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=192, rotary_dim=0, fp32_weights=True, bias=1.0),
|
||||
id="no-rope",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=64, rotary_dim=8, fp32_weights=True),
|
||||
id="minimum-rotary-dim",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_norm_rope_and_cache(dtype, case):
|
||||
check_norm_rope_and_cache(dtype=dtype, **case)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(dict(num_tokens=0), id="empty-input"),
|
||||
pytest.param(dict(use_cuda_graph=True), id="graph-auto-cache"),
|
||||
pytest.param(
|
||||
dict(use_cuda_graph=True, kv_cache_dtype="fp8"), id="graph-fp8-cache"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_empty_input_and_graph_replay(case):
|
||||
check_norm_rope_and_cache(**case)
|
||||
@@ -0,0 +1,218 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the LoRA-aware FlashInfer TRTLLM MXFP4 experts
|
||||
(`TrtLlmMxfp4LoRAExperts`) against a float64 reference."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmMxfp4LoRAExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_requested_kernel_cls,
|
||||
backend_to_kernel_cls,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# FlashInfer's TRTLLM MoE LoRA kernels only run on SM100-family GPUs.
|
||||
pytestmark = [
|
||||
pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100)
|
||||
or not has_flashinfer_trtllm_fused_moe(),
|
||||
reason="Requires an SM100-family GPU and FlashInfer TRTLLM fused MoE",
|
||||
),
|
||||
pytest.mark.skip_global_cleanup,
|
||||
]
|
||||
|
||||
E, N, K, T, TOPK = 16, 256, 512, 33, 4
|
||||
E2M1 = [0, 0.5, 1, 1.5, 2, 3, 4, 6]
|
||||
|
||||
|
||||
def _dequant_mxfp4(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
"""(E, R, K/2) packed nibbles + (E, R, K/32) e8m0 scales -> (E, R, K) f64."""
|
||||
nibbles = torch.stack([weight & 0xF, weight >> 4], dim=-1).flatten(-2).long()
|
||||
magnitude = torch.tensor(E2M1, dtype=torch.float64, device=weight.device)
|
||||
values = magnitude[nibbles & 7] * torch.where(nibbles >> 3 == 1, -1.0, 1.0)
|
||||
block_scale = torch.pow(2.0, scale.double() - 127.0)
|
||||
return values * block_scale.repeat_interleave(32, dim=-1)
|
||||
|
||||
|
||||
def _make_experts(has_bias: bool, activation=MoEActivation.SILU):
|
||||
w13 = torch.randint(0, 256, (E, 2 * N, K // 2), dtype=torch.uint8, device="cuda")
|
||||
w2 = torch.randint(0, 256, (E, K, N // 2), dtype=torch.uint8, device="cuda")
|
||||
s13 = torch.randint(118, 122, (E, 2 * N, K // 32), dtype=torch.uint8, device="cuda")
|
||||
s2 = torch.randint(118, 122, (E, K, N // 32), dtype=torch.uint8, device="cuda")
|
||||
b13 = torch.randn(E, 2 * N, device="cuda") / 4 if has_bias else None
|
||||
b2 = torch.randn(E, K, device="cuda") / 4 if has_bias else None
|
||||
k13, k2, ks13, ks2, kb13, kb2 = convert_weight_to_mxfp4_moe_kernel_format(
|
||||
Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
|
||||
None,
|
||||
w13.clone(),
|
||||
w2.clone(),
|
||||
s13.clone(),
|
||||
s2.clone(),
|
||||
None if b13 is None else b13.clone(),
|
||||
None if b2 is None else b2.clone(),
|
||||
_cache_permute_indices={},
|
||||
)
|
||||
config = make_dummy_moe_config(
|
||||
num_experts=E,
|
||||
experts_per_token=TOPK,
|
||||
hidden_dim=K,
|
||||
intermediate_size=N,
|
||||
activation=activation,
|
||||
)
|
||||
quant_config = mxfp4_w4a16_moe_quant_config(ks13, ks2, kb13, kb2)
|
||||
experts = TrtLlmMxfp4LoRAExperts(config, quant_config)
|
||||
reference = (_dequant_mxfp4(w13, s13), _dequant_mxfp4(w2, s2), b13, b2)
|
||||
return experts, (k13, k2), reference
|
||||
|
||||
|
||||
def _make_inputs():
|
||||
x = torch.randn(T, K, device="cuda").to(torch.bfloat16)
|
||||
ids = torch.stack([torch.randperm(E, device="cuda")[:TOPK] for _ in range(T)])
|
||||
weights = torch.rand(T, TOPK, device="cuda", dtype=torch.float32) + 0.1
|
||||
return x, ids.to(torch.int32), weights
|
||||
|
||||
|
||||
def _reference(reference, x, ids, weights, gate_delta=None, up_delta=None):
|
||||
"""Float64 MoE; the checkpoint w13 layout is [gate | up]."""
|
||||
w13, w2, b13, b2 = reference
|
||||
ids = ids.long()
|
||||
h = torch.einsum("tk,tjnk->tjn", x.double(), w13[ids])
|
||||
if b13 is not None:
|
||||
h = h + b13.double()[ids]
|
||||
gate, up = h[..., :N], h[..., N:]
|
||||
if gate_delta is not None:
|
||||
gate, up = gate + gate_delta.double(), up + up_delta.double()
|
||||
act = torch.nn.functional.silu(gate) * up
|
||||
out = torch.einsum("tjn,tjkn->tjk", act, w2[ids])
|
||||
if b2 is not None:
|
||||
out = out + b2.double()[ids]
|
||||
return act, (out * weights.double()[..., None]).sum(1)
|
||||
|
||||
|
||||
def _run_lora_path(experts, kernel_weights, x, ids, weights, delta):
|
||||
"""`delta` is (T, TOPK, 2I) in the [up | gate] order of apply_w13_lora."""
|
||||
lora_context = SimpleNamespace(w13_num_slices=2)
|
||||
gemm2, expert_weights, idx_map, act_permuted = experts.invoke_routed_moe(
|
||||
hidden_states=x,
|
||||
w1=kernel_weights[0],
|
||||
w2=kernel_weights[1],
|
||||
topk_ids_and_weights=(ids, weights),
|
||||
gemm1_lora_delta=experts.prepare_gemm1_lora_delta(delta, lora_context, ids),
|
||||
global_num_experts=E,
|
||||
a1q_scale=None,
|
||||
output=torch.empty(T, K, dtype=torch.bfloat16, device="cuda"),
|
||||
)
|
||||
assert act_permuted.dtype == torch.bfloat16
|
||||
act = experts._unpermute_activation(act_permuted, idx_map, T, TOPK, N)
|
||||
output = torch.empty(T, K, dtype=torch.bfloat16, device="cuda")
|
||||
no_w2_delta = torch.zeros(T, TOPK, K, dtype=torch.bfloat16, device="cuda")
|
||||
experts._finalize_with_w2_lora(
|
||||
output, gemm2, expert_weights, idx_map, no_w2_delta, T, TOPK
|
||||
)
|
||||
return act.view(T, TOPK, N), output
|
||||
|
||||
|
||||
def _rel_err(actual: torch.Tensor, expected: torch.Tensor) -> float:
|
||||
actual, expected = actual.double(), expected.double()
|
||||
return float((actual - expected).norm() / expected.norm())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_gemm1_lora_delta_matches_reference(has_bias: bool):
|
||||
set_random_seed(0)
|
||||
experts, kernel_weights, reference = _make_experts(has_bias)
|
||||
x, ids, weights = _make_inputs()
|
||||
gate_delta = (torch.randn(T, TOPK, N, device="cuda") / 2).to(torch.bfloat16)
|
||||
up_delta = (torch.randn(T, TOPK, N, device="cuda") / 2).to(torch.bfloat16)
|
||||
ref_act, ref_out = _reference(reference, x, ids, weights, gate_delta, up_delta)
|
||||
_, ref_base_out = _reference(reference, x, ids, weights)
|
||||
# The delta has to matter for the layout check below to mean anything.
|
||||
assert _rel_err(ref_out, ref_base_out) > 0.3
|
||||
|
||||
delta = torch.cat([up_delta, gate_delta], dim=-1)
|
||||
act, out = _run_lora_path(experts, kernel_weights, x, ids, weights, delta)
|
||||
assert _rel_err(act, ref_act) < 0.02
|
||||
assert _rel_err(out, ref_out) < 0.02
|
||||
|
||||
swapped = torch.cat([gate_delta, up_delta], dim=-1)
|
||||
_, out = _run_lora_path(experts, kernel_weights, x, ids, weights, swapped)
|
||||
assert _rel_err(out, ref_out) > 0.3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_bias", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_zero_delta_matches_base_path(has_bias: bool):
|
||||
set_random_seed(0)
|
||||
experts, kernel_weights, reference = _make_experts(has_bias)
|
||||
x, ids, weights = _make_inputs()
|
||||
base = torch.empty(T, K, dtype=torch.bfloat16, device="cuda")
|
||||
experts.invoke_routed_moe(
|
||||
hidden_states=x,
|
||||
w1=kernel_weights[0],
|
||||
w2=kernel_weights[1],
|
||||
topk_ids_and_weights=(ids, weights),
|
||||
gemm1_lora_delta=None,
|
||||
global_num_experts=E,
|
||||
a1q_scale=None,
|
||||
output=base,
|
||||
)
|
||||
_, ref_out = _reference(reference, x, ids, weights)
|
||||
assert _rel_err(base, ref_out) < 0.02
|
||||
|
||||
zero = torch.zeros(T, TOPK, 2 * N, dtype=torch.bfloat16, device="cuda")
|
||||
_, out = _run_lora_path(experts, kernel_weights, x, ids, weights, zero)
|
||||
assert _rel_err(out, base) < 0.02
|
||||
|
||||
|
||||
@pytest.mark.parametrize("w13_num_slices", [1, 2])
|
||||
@pytest.mark.parametrize("activation", [MoEActivation.SILU, MoEActivation.SWIGLUOAI])
|
||||
@torch.inference_mode()
|
||||
def test_prepare_gemm1_lora_delta_layout(activation, w13_num_slices: int):
|
||||
set_random_seed(0)
|
||||
experts, _, _ = _make_experts(False, activation)
|
||||
ids = _make_inputs()[1]
|
||||
gate = torch.randn(T, TOPK, N, device="cuda").to(torch.bfloat16)
|
||||
up = torch.randn(T, TOPK, N, device="cuda").to(torch.bfloat16)
|
||||
if w13_num_slices == 2:
|
||||
# What apply_w13_lora(swap_w13_slices=True) writes for per-expert LoRA.
|
||||
pair, halves = (up, gate), (up, gate)
|
||||
else:
|
||||
# A fused (3D) adapter follows the checkpoint w13 row order.
|
||||
pair, halves = (gate, up), (gate, up)
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
delta = torch.stack(pair, dim=-1).flatten(-2)
|
||||
else:
|
||||
delta = torch.cat(halves, dim=-1)
|
||||
|
||||
lora_context = SimpleNamespace(w13_num_slices=w13_num_slices)
|
||||
prepared = experts.prepare_gemm1_lora_delta(delta, lora_context, ids)
|
||||
assert torch.equal(prepared, torch.cat([up, gate], dim=-1))
|
||||
|
||||
|
||||
def test_lora_experts_are_opt_in_for_bf16_activations():
|
||||
config = make_dummy_moe_config(num_experts=E, hidden_dim=K, intermediate_size=N)
|
||||
bf16 = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16
|
||||
mxfp8 = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
assert _requested_kernel_cls(bf16, config) == backend_to_kernel_cls(bf16)
|
||||
assert _requested_kernel_cls(mxfp8, config) == backend_to_kernel_cls(mxfp8)
|
||||
|
||||
config.is_lora_enabled = True
|
||||
assert _requested_kernel_cls(bf16, config)[0] is TrtLlmMxfp4LoRAExperts
|
||||
with pytest.raises(ValueError, match="MXFP8 activations"):
|
||||
_requested_kernel_cls(mxfp8, config)
|
||||
marlin = Mxfp4MoeBackend.MARLIN
|
||||
assert _requested_kernel_cls(marlin, config) == backend_to_kernel_cls(marlin)
|
||||
@@ -110,7 +110,7 @@ def test_modernbert_models(
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -17,6 +18,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
@@ -348,3 +350,56 @@ def test_encoder_chunk_no_free_memory_falls_back_to_one():
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
|
||||
model = object.__new__(Gemma4ForConditionalGeneration)
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=16, pooling_kernel_size=3, default_output_length=280
|
||||
)
|
||||
model.vision_tower = SimpleNamespace(config=vision_config)
|
||||
model.config = SimpleNamespace(vision_config=vision_config)
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
|
||||
)
|
||||
def test_encoder_cudagraph_capture_rows_fit_token_budget(
|
||||
token_budget: int, row_tokens: int
|
||||
):
|
||||
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
|
||||
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
inputs = model.prepare_encoder_cudagraph_capture_inputs(
|
||||
token_budget=token_budget,
|
||||
max_batch_size=58,
|
||||
max_frames_per_batch=58 * 32,
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
axis_keys=(row_tokens,),
|
||||
)
|
||||
|
||||
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
|
||||
assert row_patches == row_tokens * 9
|
||||
assert num_rows == max(1, token_budget // row_tokens)
|
||||
|
||||
|
||||
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
mm_kwargs = {
|
||||
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
|
||||
"pixel_position_ids": [
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
torch.zeros(630, 2, dtype=torch.long),
|
||||
],
|
||||
}
|
||||
|
||||
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
|
||||
assert [spec.output_tokens for spec in specs] == [280, 70]
|
||||
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
|
||||
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The DeepSeek-V4/V4.1 VL wrappers stream the language model's weights to it
|
||||
as one contiguous group."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA implementations"
|
||||
)
|
||||
|
||||
|
||||
class _FakeFinalizingLanguageModel(nn.Module):
|
||||
finalizes_weights_during_load = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.tensor_a = nn.Parameter(torch.zeros(1))
|
||||
self.tensor_c = nn.Parameter(torch.zeros(1))
|
||||
self.load_calls = 0
|
||||
self.finalized_values: list[tuple[float, float]] = []
|
||||
|
||||
def load_weights(self, weights) -> set[str]:
|
||||
self.load_calls += 1
|
||||
loaded = set()
|
||||
for name, value in weights:
|
||||
getattr(self, name).data.copy_(value)
|
||||
loaded.add(name)
|
||||
self.finalized_values.append((self.tensor_a.item(), self.tensor_c.item()))
|
||||
return loaded
|
||||
|
||||
|
||||
def _v4_wrapper():
|
||||
from vllm.models.deepseek_v4.common.vl_model import (
|
||||
DeepseekV4ForConditionalGeneration,
|
||||
)
|
||||
|
||||
return DeepseekV4ForConditionalGeneration
|
||||
|
||||
|
||||
def _v41_wrapper():
|
||||
from vllm.models.deepseek_v41.nvidia.vl_model import DeepseekV41ForCausalLM
|
||||
|
||||
return DeepseekV41ForCausalLM
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wrapper", [_v4_wrapper, _v41_wrapper], ids=["v4", "v41"])
|
||||
def test_load_weights_streams_language_model_group(wrapper) -> None:
|
||||
model = object.__new__(wrapper())
|
||||
nn.Module.__init__(model)
|
||||
model.language_model = _FakeFinalizingLanguageModel()
|
||||
model.vision = nn.Module()
|
||||
model.vision.tensor_b = nn.Parameter(torch.zeros(1))
|
||||
model.hf_to_vllm_mapper = WeightsMapper()
|
||||
|
||||
def interleaved_weights():
|
||||
yield "language_model.tensor_a", torch.tensor([1.0])
|
||||
# Loaded before the stream is drained, not after.
|
||||
assert model.language_model.tensor_a.item() == 1.0
|
||||
# Splits the language model's weights unless the wrapper regroups them.
|
||||
yield "vision.tensor_b", torch.tensor([2.0])
|
||||
yield "language_model.tensor_c", torch.tensor([3.0])
|
||||
|
||||
loaded = model.load_weights(interleaved_weights())
|
||||
|
||||
assert loaded == {
|
||||
"language_model.tensor_a",
|
||||
"vision.tensor_b",
|
||||
"language_model.tensor_c",
|
||||
}
|
||||
assert model.language_model.load_calls == 1
|
||||
assert model.language_model.finalized_values == [(1.0, 3.0)]
|
||||
assert model.vision.tensor_b.item() == 2.0
|
||||
@@ -6,6 +6,7 @@ These validate that the engine-driven parser correctly handles
|
||||
Qwen3 XML-style tool calls.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -808,6 +809,136 @@ class TestArgConverter:
|
||||
assert result == {"city": "Tokyo", "expr": "x<5"}
|
||||
|
||||
|
||||
WRITE_CALL_HEAD = (
|
||||
"<tool_call>\n<function=write_file>\n"
|
||||
"<parameter=path>\n/repo/app.py\n</parameter>\n"
|
||||
"<parameter=mode>\n644\n</parameter>\n"
|
||||
"<parameter=content>\n"
|
||||
)
|
||||
WRITE_CALL_TAIL = "\n</parameter>\n</function>\n</tool_call>"
|
||||
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
|
||||
# the value, including prefixes of the closing and next-parameter tags.
|
||||
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
|
||||
|
||||
|
||||
class TestArgValueExtender:
|
||||
"""The open parameter value streams without re-running the converter."""
|
||||
|
||||
@pytest.fixture
|
||||
def write_tool(self):
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "write_file",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
"mode": {"type": "integer"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
|
||||
@pytest.mark.parametrize("parser_name", ["qwen3", "mimo"])
|
||||
def test_long_call_streams_non_streaming_arguments(
|
||||
self, mock_tokenizer, mock_request, write_tool, use_schema, parser_name
|
||||
):
|
||||
from vllm.parser.mimo import MiMoParser
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
|
||||
cls = MiMoParser if parser_name == "mimo" else Qwen3Parser
|
||||
mock_request.tools = [write_tool] if use_schema else []
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
|
||||
|
||||
def make_parser():
|
||||
return cls(mock_tokenizer, chat_template_kwargs={"enable_thinking": False})
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
make_parser(),
|
||||
mock_request,
|
||||
[text[i : i + 7] for i in range(0, len(text), 7)],
|
||||
)
|
||||
result = make_parser().extract_tool_calls(text, mock_request)
|
||||
|
||||
assert collect_tool_arguments(results) == (
|
||||
result.tool_calls[0].function.arguments
|
||||
)
|
||||
# MiMo keeps the newlines around the value; the schema coerces both.
|
||||
mode = '"\\n644\\n"' if parser_name == "mimo" else '"644"'
|
||||
if use_schema:
|
||||
mode = "644"
|
||||
assert f'"mode": {mode},' in collect_tool_arguments(results)
|
||||
|
||||
def test_conversions_do_not_grow_with_argument_length(
|
||||
self, mock_tokenizer, mock_request, write_tool
|
||||
):
|
||||
mock_request.tools = [write_tool]
|
||||
base = qwen3_config(thinking=False)
|
||||
converter = base.arg_converter
|
||||
assert converter is not None
|
||||
|
||||
def streaming_conversions(lines: int) -> int:
|
||||
calls = 0
|
||||
|
||||
def counting_converter(raw_args: str, partial: bool) -> str:
|
||||
nonlocal calls
|
||||
calls += partial
|
||||
return converter(raw_args, partial)
|
||||
|
||||
parser = ParserEngine(
|
||||
mock_tokenizer,
|
||||
parser_engine_config=dataclasses.replace(
|
||||
base, arg_converter=counting_converter
|
||||
),
|
||||
)
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
|
||||
simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[text[i : i + 4] for i in range(0, len(text), 4)],
|
||||
)
|
||||
return calls
|
||||
|
||||
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
|
||||
|
||||
def test_extender_follows_newline_trimming_and_split_end(self):
|
||||
from vllm.parser.qwen3 import _qwen3_arg_value_extender
|
||||
|
||||
extend = _qwen3_arg_value_extender("<parameter=k>")
|
||||
assert extend is not None
|
||||
deltas = ["\n", 'a"\n', "b", "\n", "</param", "eter>"]
|
||||
expected = ["", 'a\\"', "\\nb", "", "\\n</param", None]
|
||||
assert [extend(d) for d in deltas] == expected
|
||||
|
||||
def test_extender_stops_at_split_next_parameter(self):
|
||||
from vllm.parser.qwen3 import _qwen3_arg_value_extender
|
||||
|
||||
extend = _qwen3_arg_value_extender("<parameter=k>x")
|
||||
assert extend is not None
|
||||
assert [extend(d) for d in ["< para", "meter ="]] == ["< para", None]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_args",
|
||||
[
|
||||
"<parameter=k>x</parameter>",
|
||||
"<parameter=k",
|
||||
"<parameter=k>1</parameter><parameter=j>2</parameter><parameter=k>3",
|
||||
],
|
||||
ids=["closed_value", "open_name", "repeated_key_not_last"],
|
||||
)
|
||||
def test_no_extender_unless_open_value_is_last(self, raw_args):
|
||||
from vllm.parser.qwen3 import _qwen3_arg_value_extender
|
||||
|
||||
assert _qwen3_arg_value_extender(raw_args) is None
|
||||
|
||||
|
||||
class TestSchemaAwareTypeCoercion:
|
||||
"""Verify that _fix_arg_types corrects miscoerced values using the
|
||||
tool schema."""
|
||||
|
||||
@@ -2,13 +2,20 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import CompletionOutput, PromptType, RequestOutput, SamplingParams
|
||||
from vllm import (
|
||||
CompletionOutput,
|
||||
PromptType,
|
||||
RequestOutput,
|
||||
SamplingParams,
|
||||
TokensPrompt,
|
||||
)
|
||||
from vllm import logger as vllm_logger
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
@@ -17,9 +24,10 @@ from vllm.entrypoints.generate.beam_search.utils import (
|
||||
BeamSearchInstance,
|
||||
BeamSearchSequence,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import EngineInput, TokensInput
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputBackend
|
||||
|
||||
|
||||
@@ -48,8 +56,10 @@ class _Renderer:
|
||||
|
||||
|
||||
class _VllmConfig:
|
||||
use_v2_model_runner = True
|
||||
watermark_config = object()
|
||||
speculative_config = None
|
||||
model_config = SimpleNamespace(is_diffusion=False)
|
||||
|
||||
def _check_supports_watermarking(self, params: BeamSearchParams) -> bool:
|
||||
return VllmConfig._check_supports_watermarking(cast(VllmConfig, self), params)
|
||||
@@ -322,3 +332,89 @@ def test_offline_structured_beam_search_disables_internal_watermarking() -> None
|
||||
assert beam_params.watermarking is False
|
||||
assert beam_params.allowed_token_ids == [0]
|
||||
assert base_params.allowed_token_ids is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("offline", [False, True])
|
||||
@pytest.mark.parametrize("exhausted", [False, True])
|
||||
async def test_explicit_beam_allowlist_uses_exact_scores(offline, exhausted):
|
||||
prompt: TokensInput = {"type": "token", "prompt_token_ids": [1]}
|
||||
base = SamplingParams(watermarking=False)
|
||||
template = [r async for r in _EngineClient().generate(prompt, base)][0]
|
||||
template.outputs[0].logprobs[0] = dict(
|
||||
reversed(list(template.outputs[0].logprobs[0].items()))
|
||||
)
|
||||
calls = []
|
||||
if exhausted:
|
||||
for logprob in template.outputs[0].logprobs[0].values():
|
||||
logprob.logprob = float("-inf")
|
||||
|
||||
def score(params):
|
||||
assert params.logprob_token_ids == [13, 14]
|
||||
assert params.logprobs is None and params.allowed_token_ids is None
|
||||
calls.append(params)
|
||||
return template
|
||||
|
||||
class Client(_EngineClient):
|
||||
async def generate(self, prompt, params, *args, **kwargs):
|
||||
yield score(params)
|
||||
|
||||
class Offline(_OfflineServing):
|
||||
_beam_search_step = BeamSearchOfflineMixin._beam_search_step
|
||||
|
||||
def _render_and_run_requests(self, prompts, params, output_type, **kwargs):
|
||||
return [score(p) for p in params]
|
||||
|
||||
params = BeamSearchParams(beam_width=2, max_tokens=2, allowed_token_ids=[13, 14])
|
||||
ids: list[Sequence[int]]
|
||||
scores: list[float | None]
|
||||
if offline:
|
||||
result = (
|
||||
Offline().beam_search([cast(TokensPrompt, prompt)], params)[0].sequences
|
||||
)
|
||||
ids = [b.tokens[1:] for b in result]
|
||||
scores = [b.cum_logprob for b in result]
|
||||
step_scores = [step for b in result for step in b.logprobs]
|
||||
else:
|
||||
serving = _AsyncServing()
|
||||
serving.engine_client = Client()
|
||||
online_result = [r async for r in serving.beam_search(prompt, "test", params)][
|
||||
0
|
||||
]
|
||||
ids = [b.token_ids for b in online_result.outputs]
|
||||
scores = [b.cumulative_logprob for b in online_result.outputs]
|
||||
step_scores = [
|
||||
step for b in online_result.outputs for step in (b.logprobs or [])
|
||||
]
|
||||
if exhausted:
|
||||
assert len(calls) == 1
|
||||
assert ids == scores == []
|
||||
return
|
||||
assert len(calls) == 3
|
||||
assert ids[0] == [13, 13] and all(set(x) <= {13, 14} for x in ids)
|
||||
assert scores == [-6.0, -7.0]
|
||||
assert all(list(step) == [13, 14] for step in step_scores)
|
||||
assert all(score.rank is None for step in step_scores for score in step.values())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", ["empty", "grammar", "spec", "diffusion", "v1", "size", "id"]
|
||||
)
|
||||
def test_beam_allowlist_rejects_unsupported_requests(case):
|
||||
config = SimpleNamespace(
|
||||
use_v2_model_runner=case != "v1",
|
||||
speculative_config=object() if case == "spec" else None,
|
||||
model_config=SimpleNamespace(
|
||||
is_diffusion=case == "diffusion",
|
||||
max_logprobs=20,
|
||||
get_vocab_size=lambda: 256,
|
||||
),
|
||||
)
|
||||
ids = {"empty": [], "size": list(range(129)), "id": [256]}.get(case, [13])
|
||||
params = BeamSearchParams(2, 1, allowed_token_ids=ids)
|
||||
if case == "grammar":
|
||||
params.structured_outputs = StructuredOutputsParams(regex="a")
|
||||
sampling = SamplingParams(logprobs=4)
|
||||
with pytest.raises((ValueError, VLLMValidationError)):
|
||||
params.configure_logprobs(sampling, config)
|
||||
sampling._validate_logprobs(config.model_config)
|
||||
|
||||
@@ -58,7 +58,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
SlidingWindowSpec,
|
||||
SparseCacheRole,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
@@ -1945,247 +1944,102 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
|
||||
)
|
||||
|
||||
|
||||
def _spec_decode_grouping_config(method="dspark", model_type=None):
|
||||
"""Grouping config with an EAGLE-family speculative method enabled."""
|
||||
return SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False),
|
||||
cache_config=SimpleNamespace(
|
||||
get_resolved_kv_cache_layout=lambda: SimpleNamespace(
|
||||
is_block_outermost=True
|
||||
)
|
||||
),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type=model_type)),
|
||||
speculative_config=SimpleNamespace(
|
||||
method=method,
|
||||
use_eagle=lambda: True,
|
||||
use_eagle_block_drop=lambda: True,
|
||||
),
|
||||
def _spec_decode_kv_cache_groups(worker_specs, use_eagle_block_drop=True):
|
||||
config = VllmConfig(model_config=ModelConfig(max_model_len=64))
|
||||
config.cache_config.kv_cache_layout = "BLHNC"
|
||||
config.speculative_config = SimpleNamespace(
|
||||
method="dflash",
|
||||
use_eagle=lambda: True,
|
||||
use_eagle_block_drop=lambda: use_eagle_block_drop,
|
||||
use_multi_module_mtp=lambda: False,
|
||||
parallel_drafting=False,
|
||||
)
|
||||
configs = get_kv_cache_configs(config, worker_specs, [2**30] * len(worker_specs))
|
||||
return generate_scheduler_kv_cache_config(configs).kv_cache_groups
|
||||
|
||||
|
||||
def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False):
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
def _as_draft(spec):
|
||||
return replace(spec, is_draft=True)
|
||||
|
||||
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
|
||||
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to match that dtype while retaining the draft
|
||||
marker as a group boundary.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.attn.1": new_mla_spec(block_size=64, cache_dtype_str=target_dtype),
|
||||
"target.mamba.0": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
"target.mamba.1": new_mamba_spec(block_size=64, mamba_cache_mode="align"),
|
||||
}
|
||||
|
||||
def _hybrid_specs(draft=True):
|
||||
"""Qwen3.5-shaped [GDN x3, full-attn] x2, plus an MTP full-attn layer."""
|
||||
specs = {}
|
||||
for i in range(8):
|
||||
specs[f"layers.{i}"] = (
|
||||
new_kv_cache_spec()
|
||||
if i % 4 == 3
|
||||
else new_mamba_spec(mamba_cache_mode="align")
|
||||
)
|
||||
if draft:
|
||||
draft_spec = new_mla_spec(block_size=64)
|
||||
specs["draft.attn.0"] = replace(draft_spec, non_causal_multi_token_decode=True)
|
||||
specs["mtp.0"] = _as_draft(new_kv_cache_spec())
|
||||
return specs
|
||||
|
||||
|
||||
def test_draft_group_annotated_on_hybrid_general_path():
|
||||
# A drafter's MLA layer carries non_causal_multi_token_decode, so its group
|
||||
# is identifiable without keying off a model version.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "draft.attn.0" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_mamba_groups_never_flagged_even_when_draft_shares_a_group():
|
||||
# Packed uniform-type groups can contain distinct target and draft layer
|
||||
# specs; the combined group still holds volatile draft KV and must be
|
||||
# flagged. What must never happen is a Mamba group being flagged: that
|
||||
# widens its lookup window to two consecutive chunks, which align-mode
|
||||
# checkpointing never produces, zeroing every lookup.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(),
|
||||
_hybrid_specs_with_draft(draft=True, draft_shares_target_spec=True),
|
||||
)
|
||||
|
||||
for group in groups:
|
||||
if "draft.attn.0" in group.layer_names:
|
||||
assert group.is_eagle_group
|
||||
if isinstance(group.kv_cache_spec, MambaSpec):
|
||||
assert not group.is_eagle_group
|
||||
|
||||
|
||||
def test_draft_group_not_annotated_without_spec_decode():
|
||||
# The marker alone must not flag anything; the eagle semantics only apply
|
||||
# when a speculative method is actually enabled.
|
||||
config = _spec_decode_grouping_config()
|
||||
config.speculative_config = None
|
||||
groups = get_kv_cache_groups(config, _hybrid_specs_with_draft(draft=True))
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_with_mamba_warns(caplog_vllm):
|
||||
# No group carries the draft marker, so consumers fall back to
|
||||
# conservative behavior that silently breaks reuse for Mamba groups.
|
||||
# That must at least be visible.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=False)
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "no KV cache group could be identified as the draft model's" in (
|
||||
caplog_vllm.text
|
||||
)
|
||||
|
||||
|
||||
def test_unidentifiable_draft_without_mamba_does_not_warn(caplog_vllm):
|
||||
# Pure-attention models degrade gracefully under the consumers'
|
||||
# conservative fallback (a one-block hit drop at most), so the warning
|
||||
# stays silent to avoid noise on every unannotated EAGLE deployment.
|
||||
specs = {
|
||||
"target.attn.0": new_mla_spec(block_size=64),
|
||||
"target.attn.1": new_mla_spec(block_size=64),
|
||||
}
|
||||
groups = get_kv_cache_groups(_spec_decode_grouping_config(), specs)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def test_no_warning_when_draft_group_is_identified(caplog_vllm):
|
||||
get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(), _hybrid_specs_with_draft(draft=True)
|
||||
)
|
||||
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
def _deepseek_v4_specs(model_version="deepseek_v4"):
|
||||
"""DeepseekV4-shaped specs: full MLA layers plus sliding-window MLA layers
|
||||
at differing page sizes, with the MTP draft layer registered last."""
|
||||
return {
|
||||
"model.layers.0.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.1.self_attn.attn": new_mla_spec(),
|
||||
"model.layers.2.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, model_version=model_version
|
||||
),
|
||||
# The MTP block registers last, and its sliding-window size differs, so
|
||||
# it lands in a group of its own.
|
||||
"model.layers.3.self_attn.attn": new_swa_mla_spec(
|
||||
head_size=1024, sliding_window=256, model_version=model_version
|
||||
),
|
||||
}
|
||||
_DRAFT_SPEC_SHAPES = {
|
||||
"uniform": lambda: {
|
||||
"layers.0": new_kv_cache_spec(),
|
||||
"draft.0": _as_draft(new_kv_cache_spec()),
|
||||
},
|
||||
"hybrid": _hybrid_specs,
|
||||
"draft_spans_groups": lambda: {
|
||||
"layers.0": new_mla_spec(),
|
||||
"layers.1": new_swa_mla_spec(head_size=1024),
|
||||
"mtp.0": _as_draft(new_mla_spec()),
|
||||
"mtp.1": _as_draft(new_swa_mla_spec(head_size=1024)),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("method", "model_type"),
|
||||
[("mtp", "deepseek_v4"), ("dspark", "deepseek_v4"), ("dspark", "deepseek_v41")],
|
||||
"make_specs", _DRAFT_SPEC_SHAPES.values(), ids=list(_DRAFT_SPEC_SHAPES)
|
||||
)
|
||||
def test_deepseek_v4_draft_group_annotated_on_packed_path(method, model_type):
|
||||
# DeepseekV4's MTP block reuses the target's decoder layer, so its spec
|
||||
# carries no draft marker and only the positional rule can find it. This
|
||||
# pins the pre-existing behaviour that the unified annotator must preserve.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method=method, model_type=model_type),
|
||||
_deepseek_v4_specs(model_version=None),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def test_trailing_layer_fallback_applies_to_any_mtp_model():
|
||||
# The positional rule is sound for every MTP drafter, not just DeepseekV4:
|
||||
# MTP blocks reuse the target's decoder layer (no spec marker) and always
|
||||
# register after every target layer. The model_type must not gate it.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp", model_type="other"),
|
||||
_deepseek_v4_specs(),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "model.layers.3.self_attn.attn" in flagged[0].layer_names
|
||||
|
||||
|
||||
def _qwen3_5_hybrid_specs(with_mtp_layer: bool):
|
||||
"""Qwen3.5-shaped hybrid: repeating [GDN x3, full-attn x1] blocks, with
|
||||
the MTP drafter's full-attn layer (spec-identical to the target's)
|
||||
registered last."""
|
||||
specs = {}
|
||||
idx = 0
|
||||
for _ in range(2):
|
||||
for _ in range(3):
|
||||
specs[f"model.layers.{idx}.linear_attn"] = new_mamba_spec(
|
||||
mamba_cache_mode="align"
|
||||
)
|
||||
idx += 1
|
||||
specs[f"model.layers.{idx}.self_attn.attn"] = new_kv_cache_spec()
|
||||
idx += 1
|
||||
if with_mtp_layer:
|
||||
specs["mtp.layers.0.self_attn.attn"] = new_kv_cache_spec()
|
||||
return specs
|
||||
|
||||
|
||||
def test_qwen3_5_mtp_draft_group_annotated_on_hybrid_path(caplog_vllm):
|
||||
# A hybrid mamba + full-attention model with an MTP drafter that is
|
||||
# spec-indistinguishable from the target reaches the general multi-group
|
||||
# path. The trailing-layer rule must locate the draft group there so the
|
||||
# Mamba groups are not swept up by the flag-all consumer fallback.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method="mtp", model_type="qwen3_5"),
|
||||
_qwen3_5_hybrid_specs(with_mtp_layer=True),
|
||||
)
|
||||
|
||||
flagged = [g for g in groups if g.is_eagle_group]
|
||||
assert len(flagged) == 1
|
||||
assert "mtp.layers.0.self_attn.attn" in flagged[0].layer_names
|
||||
for group in groups:
|
||||
if any(
|
||||
isinstance(spec, MambaSpec)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
):
|
||||
assert not group.is_eagle_group
|
||||
assert "could be identified as the draft model's" not in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["eagle", "eagle3", "dspark"])
|
||||
def test_non_mtp_eagle_hybrid_still_warns(caplog_vllm, method):
|
||||
# Other drafters are not covered by the trailing-layer rule, so an
|
||||
# unidentifiable hybrid draft must still warn.
|
||||
groups = get_kv_cache_groups(
|
||||
_spec_decode_grouping_config(method=method, model_type="qwen3_5"),
|
||||
_qwen3_5_hybrid_specs(with_mtp_layer=True),
|
||||
)
|
||||
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
assert "could be identified as the draft model's" in caplog_vllm.text
|
||||
|
||||
|
||||
def test_trailing_layer_fallback_requires_exact_partition():
|
||||
# If the groups do not partition the layers exactly (e.g. a caller that
|
||||
# dropped or duplicated layers), the positional rule is meaningless and
|
||||
# must not fire.
|
||||
from vllm.v1.core.kv_cache_planning import _annotate_eagle_groups
|
||||
|
||||
specs = _qwen3_5_hybrid_specs(with_mtp_layer=True)
|
||||
config = _spec_decode_grouping_config(method="mtp", model_type="qwen3_5")
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
for g in groups:
|
||||
g.is_eagle_group = False
|
||||
# Remove one layer from its group: no longer an exact partition.
|
||||
trimmed = [
|
||||
KVCacheGroupSpec(
|
||||
[n for n in g.layer_names if n != "model.layers.0.linear_attn"],
|
||||
g.kv_cache_spec,
|
||||
)
|
||||
for g in groups
|
||||
@pytest.mark.parametrize("pp", [1, 2])
|
||||
def test_eagle_groups_are_the_groups_holding_draft_layers(make_specs, pp):
|
||||
specs = make_specs()
|
||||
draft_layers = {name for name, spec in specs.items() if spec.is_draft}
|
||||
untagged = {name: replace(spec, is_draft=False) for name, spec in specs.items()}
|
||||
groups = _spec_decode_kv_cache_groups([specs])
|
||||
assert [(g.layer_names, g.kv_cache_spec) for g in groups] == [
|
||||
(g.layer_names, g.kv_cache_spec)
|
||||
for g in _spec_decode_kv_cache_groups([untagged])
|
||||
]
|
||||
_annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True)
|
||||
|
||||
assert not any(g.is_eagle_group for g in trimmed)
|
||||
worker_specs = [
|
||||
{n: s for n, s in specs.items() if n not in draft_layers},
|
||||
{n: s for n, s in specs.items() if n in draft_layers},
|
||||
]
|
||||
scheduler_groups = _spec_decode_kv_cache_groups(
|
||||
[specs] if pp == 1 else worker_specs
|
||||
)
|
||||
assert [g.is_eagle_group for g in scheduler_groups] == [
|
||||
bool(draft_layers & set(g.layer_names)) for g in groups
|
||||
]
|
||||
|
||||
|
||||
def test_no_eagle_groups_without_eagle_block_drop():
|
||||
groups = _spec_decode_kv_cache_groups([_hybrid_specs()], use_eagle_block_drop=False)
|
||||
assert not any(g.is_eagle_group for g in groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"make_specs,warns",
|
||||
[
|
||||
(lambda: _hybrid_specs(draft=False), True),
|
||||
(
|
||||
lambda: {
|
||||
"layers.0": new_kv_cache_spec(),
|
||||
"draft.0": new_kv_cache_spec(),
|
||||
},
|
||||
False,
|
||||
),
|
||||
],
|
||||
ids=["untagged_hybrid", "untagged_uniform_attention"],
|
||||
)
|
||||
def test_warns_only_when_untagged_draft_meets_mamba(make_specs, warns, caplog_vllm):
|
||||
_spec_decode_kv_cache_groups([make_specs()])
|
||||
warned = "could be identified as the draft model's" in caplog_vllm.text
|
||||
assert warned == warns
|
||||
|
||||
|
||||
_GQA_SPEC = FullAttentionSpec(
|
||||
|
||||
@@ -1303,6 +1303,22 @@ def test_full_attention_merge_preserves_replicated_cache_geometry(use_mla):
|
||||
type(draft).merge([draft, replace(draft, dcp_sharded=True)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"target",
|
||||
[
|
||||
new_kv_cache_spec(),
|
||||
new_mla_spec(),
|
||||
SinkFullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32, sink_len=4
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_draft_layer_tag_is_not_part_of_the_cache_format(target):
|
||||
draft = replace(target, is_draft=True)
|
||||
assert draft == target and hash(draft) == hash(target)
|
||||
assert type(target).merge([target, draft]) == target
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_type,dcp_size,expected_width",
|
||||
[
|
||||
|
||||
@@ -929,14 +929,8 @@ class TestPushWriterNotifs:
|
||||
notif_msg = f"{request_id}:1".encode()
|
||||
w._pending_completion_notifs.put(notif_msg)
|
||||
|
||||
# transfer_topo is consulted only for the producer-side path; we
|
||||
# make it a MagicMock because the D-side branch returns early.
|
||||
w.transfer_topo = MagicMock()
|
||||
w._get_new_notifs()
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
|
||||
# Notif consumed; D-side just touches _recving_transfers.
|
||||
assert notified == set()
|
||||
assert request_id in w._recving_transfers
|
||||
|
||||
def test_get_transfer_results_evicts_completed_state(self):
|
||||
@@ -1352,12 +1346,10 @@ class TestPushWriterNegative:
|
||||
logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"),
|
||||
)
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
# Forward a completion notif for an unknown request_id.
|
||||
w._pending_completion_notifs.put(b"never-heard-of-you:1")
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
assert notified == set()
|
||||
w._get_new_notifs()
|
||||
# Did not register anywhere.
|
||||
assert "never-heard-of-you" not in w._recving_transfers
|
||||
assert w.xfer_stats.data["num_notifications_after_expiry"] == [1]
|
||||
@@ -1381,7 +1373,6 @@ class TestPushWriterNegative:
|
||||
leases of tracked P-side requests on the engine main thread, and
|
||||
ignore request IDs that aren't being tracked."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
# _handle_heartbeat reads ``self._lease_extension`` (set in the
|
||||
# real ``__init__``).
|
||||
w._lease_extension = 10
|
||||
@@ -1395,8 +1386,7 @@ class TestPushWriterNegative:
|
||||
# and another tracked one.
|
||||
w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b")
|
||||
|
||||
notified = w._get_new_notifs()
|
||||
assert notified == set()
|
||||
w._get_new_notifs()
|
||||
|
||||
# Tracked leases were renewed strictly forward in time.
|
||||
now = time.perf_counter()
|
||||
@@ -1421,20 +1411,19 @@ class TestPushPipelineParallel:
|
||||
"""Each PP stage WRITEs its own layers and sends one notif; D must
|
||||
collect pp_size notifs before reporting the recv done."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.transfer_topo = MagicMock()
|
||||
request_id = "req-pp-2"
|
||||
w._recving_metadata[request_id] = MagicMock(pp_size=2)
|
||||
notif = f"{request_id}:1".encode()
|
||||
|
||||
# First stage: counted, not yet done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
w._get_new_notifs()
|
||||
assert request_id not in w._recving_transfers
|
||||
assert w.consumer_notification_counts_by_req[request_id] == 1
|
||||
|
||||
# Second (final) stage: now reported done.
|
||||
w._pending_completion_notifs.put(notif)
|
||||
assert w._get_new_notifs() == set()
|
||||
w._get_new_notifs()
|
||||
assert request_id in w._recving_transfers
|
||||
assert request_id not in w.consumer_notification_counts_by_req
|
||||
|
||||
|
||||
@@ -110,8 +110,8 @@ def test_rocm_aiter_sampler_defers_import_when_generators_force_native(
|
||||
monkeypatch.setattr("builtins.__import__", guard_aiter_sampling_import)
|
||||
|
||||
sampler = topk_topp_sampler.TopKTopPSampler()
|
||||
logits = torch.randn(2, 8)
|
||||
k = torch.full((2,), 2, dtype=torch.int32)
|
||||
logits = torch.randn(2, 8, device=DEVICE_TYPE)
|
||||
k = torch.full((2,), 2, dtype=torch.int32, device=DEVICE_TYPE)
|
||||
generators = {0: torch.Generator(device=logits.device).manual_seed(0)}
|
||||
|
||||
token_ids, logits_to_return = sampler(logits, generators, k, None)
|
||||
|
||||
@@ -103,15 +103,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
|
||||
|
||||
target_attn_layers = {"target_attn_1": mock.MagicMock()}
|
||||
all_attn_layers = {**target_attn_layers, "draft_attn_1": mock.MagicMock()}
|
||||
target_indexer_layers: dict = {}
|
||||
all_indexer_layers: dict = {}
|
||||
|
||||
mock_get_layers.side_effect = [
|
||||
target_attn_layers,
|
||||
target_indexer_layers,
|
||||
all_attn_layers,
|
||||
all_indexer_layers,
|
||||
]
|
||||
mock_get_layers.side_effect = [target_attn_layers, all_attn_layers]
|
||||
|
||||
mock_pp_group = mock.MagicMock()
|
||||
mock_pp_group.world_size = 1
|
||||
@@ -138,6 +131,8 @@ def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_gro
|
||||
assert proposer.model.lm_head == target_model.lm_head
|
||||
# MTP shares embed_tokens with target model
|
||||
assert proposer.model.model.embed_tokens == target_model.model.embed_tokens
|
||||
assert proposer._draft_attn_layer_names == {"draft_attn_1"}
|
||||
assert all_attn_layers["draft_attn_1"].is_draft_layer is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_speculative_tokens", [1])
|
||||
|
||||
@@ -9,6 +9,7 @@ from unittest.mock import AsyncMock, MagicMock
|
||||
import pytest
|
||||
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import STREAM_FINISHED, RequestOutput
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
@@ -107,6 +108,25 @@ def make_output(request_id: str, finished: bool) -> RequestOutput:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("is_mm_encoder_only", "is_diffusion"),
|
||||
[(True, False), (False, True)],
|
||||
ids=["encoder-only", "diffusion"],
|
||||
)
|
||||
def test_resumable_requests_rejected_for_non_decoder_models(
|
||||
is_mm_encoder_only: bool, is_diffusion: bool
|
||||
):
|
||||
llm = MagicMock()
|
||||
llm.vllm_config.is_mm_encoder_only = is_mm_encoder_only
|
||||
llm.model_config.is_diffusion = is_diffusion
|
||||
|
||||
with pytest.raises(
|
||||
VLLMValidationError,
|
||||
match="Resumable requests are not supported",
|
||||
):
|
||||
AsyncLLM._validate_resumable_request(llm)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_with_async_generator():
|
||||
"""Test generate with an async input generator.
|
||||
@@ -180,6 +200,8 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
llm.vllm_config = MagicMock(is_mm_encoder_only=False)
|
||||
llm.model_config = MagicMock(is_diffusion=False)
|
||||
|
||||
final_req = MagicMock()
|
||||
final_req.request_id = "request-int"
|
||||
@@ -204,6 +226,7 @@ async def test_empty_input_stream_finishes_without_engine_request():
|
||||
def _make_streaming_llm(num_chunks: int, has_request: bool = True):
|
||||
llm = AsyncLLM.__new__(AsyncLLM)
|
||||
llm._validate_streaming_input_sampling_params = MagicMock()
|
||||
llm._validate_resumable_request = MagicMock()
|
||||
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
|
||||
llm._run_output_handler = MagicMock()
|
||||
llm._add_request = AsyncMock()
|
||||
|
||||
@@ -9,6 +9,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from tests.v1.core.utils import create_scheduler as create_async_scheduler
|
||||
from tests.v1.core.utils import create_scheduler as create_spec_scheduler
|
||||
from vllm.config import DeviceConfig, VllmConfig
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -179,6 +180,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
request_id="session",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
session.max_tokens = 2
|
||||
|
||||
@@ -195,6 +197,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
assert session.sampling_params.max_tokens == 10
|
||||
assert session.max_tokens == 10
|
||||
|
||||
session.append_output_token_ids(11)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
new_request2 = DummyRequest(
|
||||
@@ -216,6 +219,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
request_id="session",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
new_request = DummyRequest(
|
||||
@@ -246,6 +250,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=[mm_feature],
|
||||
)
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
mm_feature = MultiModalFeatureSpec(
|
||||
@@ -316,6 +321,7 @@ class TestStreamingScheduler(unittest.TestCase):
|
||||
)
|
||||
scheduler.add_request(session)
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.append_output_token_ids(10)
|
||||
session.num_computed_tokens = len(session.prompt_token_ids)
|
||||
|
||||
next_request = DummyRequest(
|
||||
@@ -716,3 +722,54 @@ def test_next_chunk_fully_computed_after_stop_on_accepted_draft(async_scheduling
|
||||
|
||||
assert list(session.prompt_token_ids) == [1, 2, 3, 10, 7, 8]
|
||||
assert output.num_scheduled_tokens["session"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_queued_before_stop", [False, True])
|
||||
def test_session_resumes_after_stop_while_preempted(chunk_queued_before_stop):
|
||||
"""A session preempted with a step in flight can stop when that step's
|
||||
output is delivered. It must stay schedulable and keep its tokens."""
|
||||
scheduler = create_async_scheduler(async_scheduling=True)
|
||||
session = DummyRequest("session", prompt_token_ids=list(range(10)))
|
||||
scheduler.add_request(session)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("session", [10]))
|
||||
in_flight = scheduler.schedule()
|
||||
next_chunk = DummyRequest("session", prompt_token_ids=[20, 21])
|
||||
if chunk_queued_before_stop:
|
||||
scheduler.add_request(next_chunk)
|
||||
|
||||
scheduler.running.remove(session)
|
||||
scheduler._preempt_request(session, timestamp=0.0)
|
||||
scheduler.update_from_output(in_flight, _model_output("session", [STOP_TOKEN]))
|
||||
# The parked variant holds a streaming-input slot until the next chunk.
|
||||
assert scheduler.num_waiting_for_streaming_input == (
|
||||
0 if chunk_queued_before_stop else 1
|
||||
)
|
||||
if not chunk_queued_before_stop:
|
||||
scheduler.add_request(next_chunk)
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert scheduler.num_waiting_for_streaming_input == 0
|
||||
expected = list(range(10)) + [10, 20, 21]
|
||||
assert list(session.all_token_ids) == expected
|
||||
assert list(session.prompt_token_ids) == expected
|
||||
assert output.num_scheduled_tokens["session"] == len(expected)
|
||||
|
||||
|
||||
def test_non_resumable_request_stays_dequeued_after_stop_while_preempted():
|
||||
"""Only resumable requests continue after a stop; a request that finishes
|
||||
while preempted must not be put back in a queue."""
|
||||
scheduler = create_async_scheduler(async_scheduling=True)
|
||||
request = DummyRequest("req", resumable=False, prompt_token_ids=list(range(10)))
|
||||
scheduler.add_request(request)
|
||||
scheduler.update_from_output(scheduler.schedule(), _model_output("req", [10]))
|
||||
in_flight = scheduler.schedule()
|
||||
|
||||
scheduler.running.remove(request)
|
||||
scheduler._preempt_request(request, timestamp=0.0)
|
||||
scheduler.update_from_output(in_flight, _model_output("req", [STOP_TOKEN]))
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
assert request not in scheduler.waiting
|
||||
assert request not in scheduler.kv_holding_waiting
|
||||
assert request not in scheduler.deferred_waiting
|
||||
assert "req" not in scheduler.schedule().num_scheduled_tokens
|
||||
|
||||
@@ -92,6 +92,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
layers[name] = SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
|
||||
get_attn_backend=lambda backend=backend: backend,
|
||||
is_draft_layer=False,
|
||||
)
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
monkeypatch.setattr(
|
||||
@@ -112,6 +113,29 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
assert all(spec.block_size == block_size for spec in specs.values())
|
||||
|
||||
|
||||
def test_get_kv_cache_spec_tags_draft_layers(monkeypatch):
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
customize_spec=AttentionBackend.customize_spec,
|
||||
get_supported_kernel_block_sizes=lambda _=None: [16],
|
||||
)
|
||||
layers = {
|
||||
name: SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _: spec,
|
||||
get_attn_backend=lambda: backend,
|
||||
is_draft_layer=name == "draft",
|
||||
)
|
||||
for name in ("target", "draft")
|
||||
}
|
||||
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
|
||||
config = SimpleNamespace(attention_config=SimpleNamespace(hisparse_config=None))
|
||||
|
||||
specs = attn_utils.get_kv_cache_spec(config)
|
||||
assert specs["draft"].is_draft and not specs["target"].is_draft
|
||||
|
||||
|
||||
class _FakeMetadataBuilder:
|
||||
def __init__(self, support: AttentionCGSupport, varlen_bound: int | None = None):
|
||||
self.support = support
|
||||
|
||||
@@ -305,3 +305,104 @@ def test_dummy_request_slot_mapping_is_pad():
|
||||
num_tokens_padded=3,
|
||||
)
|
||||
assert dummy[0].tolist() == [PAD_SLOT_ID] * 3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def populated_block_tables():
|
||||
tables = BlockTables([16, 32], 8, 64, [3, 5], torch.device("cuda"))
|
||||
for req_index in range(8):
|
||||
tables.append_block_ids(
|
||||
req_index, ([req_index + 1] * 3, [req_index + 1] * 5), overwrite=True
|
||||
)
|
||||
tables.apply_staged_writes()
|
||||
tables.gather_block_tables(torch.arange(8, device="cuda"), num_reqs_padded=8)
|
||||
return tables
|
||||
|
||||
|
||||
@pytest.mark.parametrize("previous_dummy", [False, True])
|
||||
@pytest.mark.parametrize("next_dummy", [False, True])
|
||||
def test_shared_block_tables_clear_rows_from_larger_batch(
|
||||
populated_block_tables, previous_dummy, next_dummy
|
||||
):
|
||||
"""A smaller target batch must leave no stale rows for larger draft graphs."""
|
||||
tables = populated_block_tables
|
||||
if previous_dummy:
|
||||
for table in tables.get_dummy_block_tables(8):
|
||||
table.fill_(99) # Profiling supplies valid context/state slots.
|
||||
ptrs = [table.data_ptr() for table in tables.input_block_tables]
|
||||
idx = torch.tensor([7, 2, 0, 5, 1], device="cuda")
|
||||
for num_reqs in (3, 5, 0):
|
||||
if next_dummy:
|
||||
out = tables.get_dummy_block_tables(num_reqs)
|
||||
num_rows = num_reqs
|
||||
else:
|
||||
num_rows = num_reqs + 1
|
||||
out = tables.gather_block_tables(idx[:num_reqs], num_rows)
|
||||
for table, view, ptr in zip(tables.input_block_tables, out, ptrs):
|
||||
assert view.shape == (num_rows, table.shape[1])
|
||||
assert table.data_ptr() == ptr
|
||||
if num_rows:
|
||||
assert view.data_ptr() == ptr
|
||||
assert torch.count_nonzero(table[num_reqs:]).item() == 0
|
||||
if next_dummy:
|
||||
assert torch.count_nonzero(view).item() == 0
|
||||
view.fill_(99)
|
||||
else:
|
||||
expected = (idx[:num_reqs, None] + 1).expand(-1, table.shape[1])
|
||||
torch.testing.assert_close(view[:num_reqs], expected.to(torch.int32))
|
||||
|
||||
|
||||
def test_external_block_table_output_preserves_shared_rows(populated_block_tables):
|
||||
"""PCP-local gathers must neither clear shared rows nor reset their tracking."""
|
||||
tables = populated_block_tables
|
||||
previous = [table.clone() for table in tables.input_block_tables]
|
||||
out = tuple(torch.full_like(table[:3], -1) for table in previous)
|
||||
out_ptrs = torch.tensor(
|
||||
[table.data_ptr() for table in out], dtype=torch.uint64, device="cuda"
|
||||
)
|
||||
idx = torch.tensor([7, 2], device="cuda")
|
||||
result = tables.gather_block_tables(idx, 3, out=out, out_ptrs=out_ptrs)
|
||||
for table, before, view in zip(tables.input_block_tables, previous, result):
|
||||
torch.testing.assert_close(table, before)
|
||||
assert view[:, 0].tolist() == [8, 3, 0]
|
||||
tables.gather_block_tables(idx[:1], num_reqs_padded=1)
|
||||
for table in tables.input_block_tables:
|
||||
assert torch.count_nonzero(table[1:]).item() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_cuda_graph", [False, True])
|
||||
def test_larger_draft_batch_uses_null_kpool_slots(
|
||||
populated_block_tables, use_cuda_graph
|
||||
):
|
||||
"""Draft padding must not resolve to another request's physical KPool slots."""
|
||||
from vllm.v1.attention.backends.mla.indexer import compute_kpool_tail_slot_mapping
|
||||
|
||||
tables = populated_block_tables
|
||||
idx = torch.tensor([2, 0, 4, 1, 3], device="cuda")
|
||||
query_start_loc = torch.arange(9, dtype=torch.int32, device="cuda").clamp_max_(5)
|
||||
slot_mapping = torch.full((8,), -1, dtype=torch.int64, device="cuda")
|
||||
positions = torch.arange(8, dtype=torch.int64, device="cuda")
|
||||
|
||||
def compute_slots():
|
||||
return compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
tables.input_block_tables[0],
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_actual_tokens=8,
|
||||
num_reqs=8,
|
||||
kpool=8,
|
||||
)
|
||||
|
||||
if use_cuda_graph:
|
||||
compute_slots()
|
||||
torch.accelerator.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
slots = compute_slots()
|
||||
tables.gather_block_tables(idx, num_reqs_padded=5)
|
||||
if use_cuda_graph:
|
||||
graph.replay()
|
||||
else:
|
||||
slots = compute_slots()
|
||||
assert slots.tolist() == [24, 9, 42, 19, 36, -1, -1, -1]
|
||||
|
||||
@@ -22,7 +22,12 @@ def _make_connector(
|
||||
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
|
||||
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
|
||||
backend.wait_for_save.side_effect = lambda: events.append("wait")
|
||||
backend.get_transfer_results.return_value = KVConnectorTransferResults()
|
||||
|
||||
def get_transfer_results(_finished_req_ids):
|
||||
events.append("results")
|
||||
return KVConnectorTransferResults()
|
||||
|
||||
backend.get_transfer_results.side_effect = get_transfer_results
|
||||
backend.get_block_ids_with_load_errors.return_value = set()
|
||||
backend.get_kv_connector_stats.return_value = None
|
||||
backend.get_kv_connector_kv_cache_events.return_value = None
|
||||
@@ -73,7 +78,7 @@ def test_load_start_phase(
|
||||
)
|
||||
|
||||
connector.post_forward(set())
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
|
||||
assert kwargs["request_state_indices"] is request_indices
|
||||
@@ -94,4 +99,21 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
connector.no_forward(_scheduler_output(False))
|
||||
|
||||
assert events == ["handle", "bind", "start", "wait", "clear"]
|
||||
assert events == ["handle", "bind", "start", "wait", "results", "clear"]
|
||||
|
||||
|
||||
def test_deferred_finalize_collects_results_before_saving(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""With a drafter, the transfer results are collected right after the
|
||||
target forward, and the saves and metadata are finalized after the
|
||||
drafter writes its KV."""
|
||||
events: list[str] = []
|
||||
connector = _make_connector(monkeypatch, events)
|
||||
connector.pre_forward(_scheduler_output(False))
|
||||
|
||||
connector.post_forward(set(), defer_finalize=True)
|
||||
assert events == ["handle", "bind", "start", "results"]
|
||||
|
||||
connector.finalize_step()
|
||||
assert events == ["handle", "bind", "start", "results", "wait", "clear"]
|
||||
|
||||
@@ -88,6 +88,7 @@ def _make_runner(
|
||||
"""Stub model runner exposing only what the warmup entry points read."""
|
||||
return SimpleNamespace(
|
||||
num_speculative_steps=num_spec_steps,
|
||||
speculative_config=SimpleNamespace() if num_spec_steps else None,
|
||||
adaptive_verification=None,
|
||||
rejection_sampler=None,
|
||||
decode_query_len=num_spec_steps + 1,
|
||||
|
||||
@@ -74,6 +74,7 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
runner = SimpleNamespace(
|
||||
adaptive_verification=manager,
|
||||
rejection_sampler=rejection_sampler,
|
||||
speculative_config=SimpleNamespace(),
|
||||
)
|
||||
|
||||
def run_steps(model_runner, execute, sample, greedy=False):
|
||||
@@ -91,3 +92,27 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
|
||||
assert runner.adaptive_verification is manager
|
||||
assert manager.cost_tables is None
|
||||
assert rejection_sampler.enable_adaptive_verification
|
||||
|
||||
|
||||
@pytest.mark.parametrize("speculative", [False, True])
|
||||
def test_kernel_warmup_runs_same_rounds_on_every_pp_stage(monkeypatch, speculative):
|
||||
"""PP receivers need a matching sender for every speculative warmup round."""
|
||||
rounds = []
|
||||
monkeypatch.setattr(
|
||||
warmup,
|
||||
"_warmup_kernels",
|
||||
lambda runner, execute, sample, greedy=False: rounds.append(greedy),
|
||||
)
|
||||
for is_last_pp_rank in (False, True):
|
||||
rounds.clear()
|
||||
runner = SimpleNamespace(
|
||||
adaptive_verification=None,
|
||||
speculative_config=SimpleNamespace() if speculative else None,
|
||||
rejection_sampler=(
|
||||
SimpleNamespace(enable_adaptive_verification=False)
|
||||
if is_last_pp_rank and speculative
|
||||
else None
|
||||
),
|
||||
)
|
||||
warmup.warmup_kernels(runner, _fail, _fail)
|
||||
assert rounds == ([False, True] if speculative else [False])
|
||||
|
||||
Reference in New Issue
Block a user