[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
@@ -1,148 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import torch
from torch import nn
from vllm.models.deepseek_v32.nvidia import model as deepseek_v32_model
from vllm.models.deepseek_v32.nvidia import mtp as deepseek_v32_mtp
class _IdentityNorm(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
self.variance_epsilon = 1e-5
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
):
if residual is None:
return hidden_states
return hidden_states, residual
class _RecordingModule(nn.Module):
def __init__(self) -> None:
super().__init__()
self.num_tokens = 0
def forward(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
self.num_tokens = hidden_states.shape[0]
return hidden_states
class _RecordingProjection(nn.Module):
def __init__(self) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(2, 4), requires_grad=False)
self.num_tokens = 0
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
self.num_tokens = hidden_states.shape[0]
return hidden_states[:, :2]
class _SequenceParallelMTPBlock:
use_sequence_parallel = True
def __call__(
self,
*,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
):
assert residual is None
return hidden_states * 2, hidden_states * 3
def _mock_sequence_parallel_collectives(monkeypatch, module):
monkeypatch.setattr(
module,
"sp_reduce_scatter",
lambda tensor: tensor.chunk(2, dim=0)[0],
raising=False,
)
monkeypatch.setattr(
module,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
raising=False,
)
monkeypatch.setattr(
module,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
def test_decoder_layer_keeps_dense_states_sequence_sharded(monkeypatch):
layer = object.__new__(deepseek_v32_model.DeepseekV32DecoderLayer)
nn.Module.__init__(layer)
layer.use_sequence_parallel = True
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.self_attn = _RecordingModule()
layer.mlp = _RecordingModule()
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_model)
positions = torch.arange(3)
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
hidden_states = deepseek_v32_model.sp_shard(full_hidden_states)
hidden_states, residual = layer(positions, hidden_states, residual=None)
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.self_attn.num_tokens == 3
assert layer.mlp.num_tokens == 2
hidden_states, residual = layer(positions, hidden_states, residual)
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.self_attn.num_tokens == 3
assert layer.mlp.num_tokens == 2
def test_mtp_projects_sequence_shard_and_restores_full_output(monkeypatch):
layer = object.__new__(deepseek_v32_mtp.DeepseekV32MultiTokenPredictorLayer)
nn.Module.__init__(layer)
layer.enorm = _IdentityNorm()
layer.hnorm = _IdentityNorm()
layer.eh_proj = _RecordingProjection()
layer._eh_plan = None
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
norm = Mock(
side_effect=lambda hidden_states, residual: (hidden_states + residual, None)
)
object.__setattr__(layer, "shared_head", SimpleNamespace(norm=norm))
monkeypatch.setattr(
deepseek_v32_mtp,
"fused_eh_norm",
lambda positions, inputs_embeds, previous_hidden_states, *args: torch.cat(
[inputs_embeds, previous_hidden_states], dim=-1
),
)
monkeypatch.setattr(deepseek_v32_mtp, "run_glm52_plan", lambda *args: None)
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_mtp)
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
hidden_states, recycled_hidden_states = layer(
input_ids=torch.zeros(3, dtype=torch.long),
positions=torch.arange(3),
previous_hidden_states=torch.zeros_like(inputs_embeds),
inputs_embeds=inputs_embeds,
)
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
expected = torch.cat([sharded_states * 5, sharded_states * 5])[:3]
assert layer.eh_proj.num_tokens == 2
torch.testing.assert_close(hidden_states, expected)
torch.testing.assert_close(recycled_hidden_states, expected)
norm.assert_called_once()
-432
View File
@@ -1,432 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash KDA recurrent (decode) kernel.
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
token-strided rather than contiguous. Both backends must match a pure-PyTorch
recurrence, read these slices in place and reject unsupported layouts.
"""
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_rocm():
from vllm.models.glm5next.amd.ops.third_party.kda import fused_recurrent_kda
else:
from vllm.models.glm5next.nvidia.ops.third_party.kda import fused_recurrent_kda
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
)
H, D = 16, 128
LOWER_BOUND = -5.0
def naive_recurrent_kda(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
a_log: torch.Tensor,
g_bias: torch.Tensor,
state: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""fp32 reference for one sequence: ``[T, H, D]`` inputs, ``[H, D, D]``
(v-major) state; mirrors the kernel's in-kernel gate, beta sigmoid and
q/k l2norm.
"""
q, k, v, raw_g, raw_beta = (x.float() for x in (q, k, v, raw_g, raw_beta))
q = q / torch.sqrt(q.square().sum(-1, keepdim=True) + 1e-6) * D**-0.5
k = k / torch.sqrt(k.square().sum(-1, keepdim=True) + 1e-6)
gate = LOWER_BOUND * torch.sigmoid(a_log.exp()[:, None] * (raw_g + g_bias))
beta = torch.sigmoid(raw_beta)
s = state.clone()
out = torch.empty_like(v)
for t in range(q.shape[0]):
s = s * gate[t].exp()[:, None, :]
u = beta[t][:, None] * (v[t] - torch.einsum("hvk,hk->hv", s, k[t]))
s = s + u[:, :, None] * k[t][:, None, :]
out[t] = torch.einsum("hvk,hk->hv", s, q[t])
return out, s
def make_inputs(num_seqs: int, query_len: int, device: torch.device):
"""Token-strided q/k/v/beta as the decode path produces them: column
slices of a merged ``[T, q|k|v]`` conv output and of the fused
``[T, qkv|beta|f_a|g_a]`` projection.
"""
T, proj = num_seqs * query_len, H * D
qkv = torch.randn(T, 3 * proj, dtype=torch.bfloat16, device=device)
projected = torch.randn(
T, 3 * proj + H + 2 * D, dtype=torch.bfloat16, device=device
)
q, k, v = (x.view(1, T, H, D) for x in qkv.split(proj, dim=-1))
beta = projected[:, 3 * proj : 3 * proj + H].unsqueeze(0)
# (A size-1 token dim gets an arbitrary stride from `view`.)
assert T == 1 or (q.stride(1) == 3 * proj and beta.stride(1) == projected.stride(0))
inputs = dict(
q=q,
k=k,
v=v,
g=torch.randn(1, T, H, D, dtype=torch.bfloat16, device=device),
beta=beta,
a_log=0.5 * torch.randn(H, dtype=torch.float32, device=device),
g_bias=0.1 * torch.randn(H * D, dtype=torch.float32, device=device),
cu_seqlens=torch.arange(0, T + 1, query_len, dtype=torch.int32, device=device),
)
# Slot 0 is NULL_BLOCK_ID; sequences own random distinct slots (one per
# token in the spec-decode layout).
slots = torch.randperm(T, device=device).to(torch.int32) + 1
if query_len == 1:
inputs["ssm_state_indices"] = slots
else:
inputs["ssm_state_indices"] = slots.view(num_seqs, query_len)
inputs["num_accepted_tokens"] = torch.randint(
1, query_len + 1, (num_seqs,), dtype=torch.int32, device=device
)
state = torch.randn(T + 1, H, D, D, dtype=torch.float32, device=device)
return inputs, state
def run_kernel(inputs: dict, state: torch.Tensor) -> torch.Tensor:
out, _ = fused_recurrent_kda(
**inputs,
initial_state=state,
use_qk_l2norm_in_kernel=True,
sigmoid_beta=True,
compute_gate=True,
lower_bound=LOWER_BOUND,
)
return out
@pytest.mark.parametrize(
("num_seqs", "query_len"), [(1, 1), (7, 1), (3, 3)], ids=["1x1", "7x1", "3x3"]
)
@torch.inference_mode()
def test_fused_recurrent_kda_matches_reference(num_seqs: int, query_len: int):
torch.manual_seed(0)
device = torch.device("cuda")
inputs, state = make_inputs(num_seqs, query_len, device)
expected_state = state.clone()
out = run_kernel(inputs, state)
indices = inputs["ssm_state_indices"].view(num_seqs, query_len)
accepted = inputs.get("num_accepted_tokens")
expected = torch.empty_like(out[0], dtype=torch.float32)
for n in range(num_seqs):
first = indices[n, 0 if accepted is None else accepted[n] - 1]
s = expected_state[first]
for t in range(query_len):
tok = slice(n * query_len + t, n * query_len + t + 1)
expected[tok], s = naive_recurrent_kda(
inputs["q"][0, tok],
inputs["k"][0, tok],
inputs["v"][0, tok],
inputs["g"][0, tok],
inputs["beta"][0, tok],
inputs["a_log"],
inputs["g_bias"].view(H, D),
s,
)
expected_state[indices[n, t]] = s
torch.testing.assert_close(out[0].float(), expected, rtol=1e-2, atol=1e-3)
used = indices.flatten().long()
torch.testing.assert_close(state[used], expected_state[used], rtol=1e-5, atol=1e-5)
@pytest.mark.parametrize(
("num_seqs", "query_len"), [(7, 1), (3, 3)], ids=["7x1", "3x3"]
)
@torch.inference_mode()
def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
num_seqs: int, query_len: int
):
torch.manual_seed(0)
device = torch.device("cuda")
inputs, state = make_inputs(num_seqs, query_len, device)
for name in ("q", "k", "v", "beta"):
assert not inputs[name].is_contiguous()
contiguous = {
name: x.contiguous() if name in ("q", "k", "v", "beta") else x
for name, x in inputs.items()
}
state_ref = state.clone()
out_ref = run_kernel(contiguous, state_ref)
out = run_kernel(inputs, state)
torch.testing.assert_close(out, out_ref, rtol=0, atol=0)
torch.testing.assert_close(state, state_ref, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Both implementations require zero-copy token-strided layouts",
)
@torch.inference_mode()
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
"""Layouts the token-stride addressing cannot express must fail loudly
rather than read the wrong tokens: a batch slice of a wider buffer
(``stride(0) != T * stride(1)``), overlapping tokens, and a head-strided
(transposed) block.
"""
torch.manual_seed(0)
device = torch.device("cuda")
inputs, state = make_inputs(1, 1, device)
T = 4
base = torch.randn(4, T, 2 * H * D, dtype=torch.bfloat16, device=device)
bad_q = {
"batch slice": base[::2, :, : H * D].view(2, T, H, D),
"overlapping tokens": base[:1, :, : H * D].as_strided(
(1, T, H, D), (0, D, D, 1)
),
"head-strided": base[:1, :, : H * D].view(1, T, D, H).transpose(2, 3),
}
for q in bad_q.values():
broken = dict(inputs, q=q, k=q, v=q)
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
with pytest.raises(AssertionError, match=r"torch.Size"):
run_kernel(broken, state)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="The large-grid fix is NVIDIA-only"
)
@pytest.mark.parametrize("num_seqs,num_heads", [(1024, 64), (2048, 48)])
def test_fused_recurrent_kda_more_than_65535_programs(num_seqs: int, num_heads: int):
"""Regression test for the launch grid.
GLM-5.3-Flash has 64 KDA value heads; at TP=1 (every DP rank) a decode
batch of 1024 sequences needs 1024 * 64 = 65536 (sequence, head)
programs. The kernel used to put that product into gridDim.z, whose CUDA
limit is 65535, so the launch failed with ``Triton Error [CUDA]: invalid
argument`` at CUDA-graph capture and DP deployments could not start. The
grid now carries the product in gridDim.x. Small head dims keep the test
cheap; the shape that matters is the number of programs. Every (sequence,
head) pair is independent, so the full batch must be bitwise identical to
the same batch run in two halves.
"""
torch.manual_seed(0)
dev = torch.device("cuda")
head_dim = 16
pool = num_seqs + 8
q = torch.randn(1, num_seqs, num_heads, head_dim, device=dev, dtype=torch.bfloat16)
k = torch.randn_like(q)
v = torch.randn_like(q)
g = torch.randn_like(q)
beta = torch.randn(1, num_seqs, num_heads, device=dev, dtype=torch.bfloat16)
state = torch.randn(
pool, num_heads, head_dim, head_dim, device=dev, dtype=torch.float32
)
cu = torch.arange(0, num_seqs + 1, device=dev, dtype=torch.int32)
# slot 0 is NULL_BLOCK_ID and is skipped by the kernel: use slots 1..
idx = (torch.randperm(pool - 1, device=dev)[:num_seqs] + 1).to(torch.int32)
a_log = torch.randn(num_heads, device=dev, dtype=torch.float32)
g_bias = torch.randn(num_heads, head_dim, device=dev, dtype=torch.float32)
def run(sl: slice, cu_: torch.Tensor, st: torch.Tensor, out: torch.Tensor):
fused_recurrent_kda(
q=q[:, sl],
k=k[:, sl],
v=v[:, sl],
g=g[:, sl],
beta=beta[:, sl],
initial_state=st,
use_qk_l2norm_in_kernel=True,
cu_seqlens=cu_,
ssm_state_indices=idx[sl],
sigmoid_beta=True,
a_log=a_log,
g_bias=g_bias,
compute_gate=True,
lower_bound=-5.0,
out=out,
)
full_state = state.clone()
full_out = torch.zeros_like(k)
run(slice(0, num_seqs), cu, full_state, full_out)
half = num_seqs // 2
ref_state = state.clone()
ref_out = torch.zeros_like(k)
for s, e in ((0, half), (half, num_seqs)):
cu_h = torch.arange(0, e - s + 1, device=dev, dtype=torch.int32)
part = torch.zeros_like(k[:, s:e])
run(slice(s, e), cu_h, ref_state, part)
ref_out[:, s:e] = part
torch.accelerator.synchronize()
assert torch.equal(full_out, ref_out)
assert torch.equal(full_state, ref_state)
@pytest.mark.parametrize("offset", [16, 144, 256])
@pytest.mark.parametrize("dim_first", [False, True])
@pytest.mark.parametrize("num_spec", [0, 3])
@torch.inference_mode()
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
"""Restoring both cached states must reproduce an uninterrupted prefill."""
from types import SimpleNamespace
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.kda_checkpoint import (
FlashKDAPrefillCheckpointExporter,
)
from vllm.models.glm5next.common import kda
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
from vllm.v1.worker.workspace import WorkspaceManager
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
pytest.skip("requires FlashKDA")
import vllm._flashkda_C # noqa: F401
torch.manual_seed(42)
device = torch.device("cuda")
length, heads = 273, 2
width, state_len = 3 * heads * D, 3 + num_spec
layer = object.__new__(kda.Glm5NextLinearAttention)
torch.nn.Module.__init__(layer)
layer.prefix = "layer.0"
layer.local_num_heads = heads
layer.head_dim = D
layer.conv_size = 4
layer.local_projection_size = heads * D
layer.kda_safe_gate = True
layer.kda_lower_bound = -5.0
layer.kda_prefill_backend = "flashkda"
layer._conv_state_dim_first = dim_first
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
layer.q_conv1d = SimpleNamespace(bias=None)
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
state_shape = (2, heads, D, D)
layer._flashkda_buffer_specs = (
(state_shape, torch.float32),
(state_shape, torch.float32),
(
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
torch.uint8,
),
((1, 2 * length, heads, D), torch.bfloat16),
)
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
workspace = WorkspaceManager(device)
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
conv_shape = (width, state_len) if dim_first else (state_len, width)
num_slots = 5 + (num_spec + 1 if num_spec else 0)
conv_storage = torch.zeros(
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
)
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
recurrent_storage = torch.zeros(
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
)
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
layer.kv_cache = (conv, recurrent)
projected = torch.randn(
2 * length, width + heads, device=device, dtype=torch.bfloat16
)
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
raw_before = raw_qkv.clone()
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
def run(qkv, g, b, slots, has_initial, checkpoint=None):
n = qkv.shape[0] // 2
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
cu_cpu, device=device
)
metadata = GDNAttentionMetadata(
num_prefills=2,
num_prefill_tokens=2 * n,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=0,
num_spec_decode_tokens=0,
num_actual_tokens=2 * n,
has_initial_state=torch.tensor(has_initial, device=device),
non_spec_query_start_loc=cu_cpu.to(device),
non_spec_state_indices_tensor=torch.tensor(
slots, dtype=torch.int32, device=device
),
checkpoint=checkpoint,
nums_dict=nums,
batch_ptr=batch_ptr,
token_chunk_offset_ptr=chunk_offsets,
)
monkeypatch.setattr(
kda,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
)
spec_tokens = num_spec + 1 if num_spec else 0
if spec_tokens:
qkv = torch.cat([qkv[:spec_tokens], qkv])
g = torch.cat([g[:, :spec_tokens], g], dim=1)
b = torch.cat([b[:, :spec_tokens], b], dim=1)
metadata.num_spec_decodes = 1
metadata.num_spec_decode_tokens = spec_tokens
metadata.num_actual_tokens += spec_tokens
metadata.spec_sequence_masks = torch.tensor(
[True, False, False], device=device
)
metadata.spec_query_start_loc = torch.tensor(
[0, spec_tokens], dtype=torch.int32, device=device
)
metadata.spec_state_indices_tensor = torch.arange(
5, num_slots, dtype=torch.int32, device=device
).unsqueeze(0)
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
metadata.non_spec_token_indx = torch.arange(
spec_tokens, qkv.shape[0], device=device
)
metadata.num_accepted_tokens = torch.ones(
1, dtype=torch.int32, device=device
)
out = torch.empty(
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
)
layer._forward(qkv, g, b, out)
return out[:, spec_tokens:]
checkpoint = MambaPrefillCheckpointMetadata(
torch.tensor([offset, 0], dtype=torch.int32, device=device),
torch.tensor([3, 0], dtype=torch.int32, device=device),
)
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
expected_state = recurrent[1].clone()
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
torch.testing.assert_close(raw_qkv, raw_before)
conv_view = conv if dim_first else conv.transpose(-1, -2)
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
run(
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
[1, 2],
[False, False],
)
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
suffix = slice(offset, length)
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
conv[4].copy_(saved_conv)
recurrent[4].copy_(saved_recurrent)
actual = run(qkv, g, b, [3, 4], [True, True])
torch.testing.assert_close(
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
)
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``--mamba-ssm-cache-dtype`` must reach GLM-5.3-Flash's KDA recurrent state
(it used to be dropped, leaving the state float32 unconditionally)."""
from types import SimpleNamespace
import pytest
import torch
from vllm.models.glm5next.common.kda import Glm5NextLinearAttention
from vllm.models.glm5next.common.model import (
Glm5NextForCausalLM,
Glm5NextForConditionalGeneration,
)
@pytest.mark.parametrize(
("ssm_dtype", "expected"),
[("auto", torch.float32), ("float32", torch.float32), ("bfloat16", torch.bfloat16)],
)
def test_kda_recurrent_state_dtype_follows_ssm_cache_dtype(ssm_dtype, expected):
model_config = SimpleNamespace(dtype=torch.bfloat16)
cache_config = SimpleNamespace(
mamba_cache_dtype="auto",
mamba_ssm_cache_dtype=ssm_dtype,
use_recoverssm=False,
)
vllm_config = SimpleNamespace(model_config=model_config, cache_config=cache_config)
want = (torch.bfloat16, expected)
assert Glm5NextForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == want
assert (
Glm5NextForConditionalGeneration.get_mamba_state_dtype_from_config(vllm_config)
== want
)
# The layer's own dtype must agree with the model-level one, since the
# KV-cache spec and the kernels read them separately.
layer = Glm5NextLinearAttention.__new__(Glm5NextLinearAttention)
layer.model_config, layer.cache_config = model_config, cache_config
assert layer.get_state_dtype() == want
@@ -1,114 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash sequence-parallel MoE layout.
With DP > 1, TP > 1 and expert parallelism, ``Glm5NextModel`` shards the token
dimension across the TP group once at the model entry and every layer runs its
MLP on that shard. A module that still does tensor-parallel collectives there
sums the partial results of *different* tokens held by different ranks, so the
dense MLP of the first ``first_k_dense_replace`` layers must hold replicated
weights and run without any collective, like the shared experts already do.
"""
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.model_executor import parameter
from vllm.model_executor.layers import linear
from vllm.models.glm5next.common import model as glm_model
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
class _Attention(nn.Module):
def __init__(self, *args, **kwargs) -> None:
super().__init__()
self.o_proj = SimpleNamespace(reduce_results=True)
def _fake_tensor_parallel_world(monkeypatch) -> None:
"""Pretend to be rank 1 of a TP=2 group whose all-reduce must never run."""
def _forbidden_all_reduce(*args, **kwargs):
raise AssertionError("tensor-parallel all-reduce on a sequence-parallel shard")
for module in (linear, parameter):
monkeypatch.setattr(
module, "get_tensor_model_parallel_world_size", lambda: TP_SIZE
)
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 1)
monkeypatch.setattr(
linear, "tensor_model_parallel_all_reduce", _forbidden_all_reduce
)
def _reference_mlp(mlp: glm_model.Glm5NextMLP, x: torch.Tensor) -> torch.Tensor:
gate_up = x @ mlp.gate_up_proj.weight.T
gate, up = gate_up.split(INTERMEDIATE, dim=-1)
return (torch.nn.functional.silu(gate) * up) @ mlp.down_proj.weight.T
def _assert_runs_locally_on_shard(mlp: glm_model.Glm5NextMLP) -> None:
device = "cuda" if torch.cuda.is_available() else "cpu"
mlp.to(device)
assert mlp.gate_up_proj.weight.shape == (2 * INTERMEDIATE, HIDDEN)
assert mlp.down_proj.weight.shape == (HIDDEN, INTERMEDIATE)
torch.manual_seed(0)
for param in mlp.parameters():
param.data.normal_()
shard = torch.randn(3, HIDDEN, device=device)
torch.testing.assert_close(mlp(shard), _reference_mlp(mlp, shard))
def test_dense_mlp_runs_locally_on_the_sequence_parallel_shard(
monkeypatch, default_vllm_config
):
_fake_tensor_parallel_world(monkeypatch)
sharded = glm_model.Glm5NextMLP(HIDDEN, INTERMEDIATE, "silu")
assert sharded.down_proj.tp_size == TP_SIZE
assert sharded.gate_up_proj.weight.shape == (2 * INTERMEDIATE // TP_SIZE, HIDDEN)
_assert_runs_locally_on_shard(
glm_model.Glm5NextMLP(HIDDEN, INTERMEDIATE, "silu", is_sequence_parallel=True)
)
@pytest.mark.parametrize("use_sequence_parallel_moe", [True, False])
def test_dense_layer_mlp_follows_the_sequence_parallel_layout(
monkeypatch, default_vllm_config, use_sequence_parallel_moe: bool
):
_fake_tensor_parallel_world(monkeypatch)
monkeypatch.setattr(glm_model, "Glm5NextLinearAttention", _Attention)
monkeypatch.setattr(glm_model, "Glm5NextMLAAttention", _Attention)
config = Glm5NextTextConfig(
hidden_size=HIDDEN,
intermediate_size=INTERMEDIATE,
num_hidden_layers=2,
first_k_dense_replace=1,
n_routed_experts=4,
layer_types=["linear_attention", "deepseek_sparse_attention"],
mhc=False,
)
vllm_config = SimpleNamespace(
cache_config=None,
quant_config=None,
parallel_config=SimpleNamespace(
use_sequence_parallel_moe=use_sequence_parallel_moe
),
kernel_config=SimpleNamespace(enable_jit_warmup=False),
)
layer = glm_model.Glm5NextDecoderLayer(vllm_config, config, layer_idx=0)
assert isinstance(layer.mlp, glm_model.Glm5NextMLP)
assert layer.self_attn.o_proj.reduce_results is not use_sequence_parallel_moe
if use_sequence_parallel_moe:
_assert_runs_locally_on_shard(layer.mlp)
else:
assert layer.mlp.down_proj.tp_size == TP_SIZE
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the GLM kpool indexer's top-k backend wiring."""
import pytest
import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.platforms import current_platform
def _require_deep_gemm() -> None:
from vllm.utils.deep_gemm import has_deep_gemm
if not has_deep_gemm():
pytest.skip("kpool indexer requires DeepGEMM")
def _build(indexer_cls, backend: str):
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
return indexer_cls(
k_cache=None,
quant_block_size=128,
scale_fmt="ue8m0",
topk_tokens=2048,
head_dim=128,
max_pool_len=4096,
max_total_seq_len=8192,
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
backend: str,
) -> None:
"""The AMD kpool indexer must go through the same dispatcher, so the
AITER decode top-k is reachable from kernel_config."""
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize(
"configured,full_cudagraph,expected",
[
("auto", False, "aiter"),
("auto", True, "auto"),
("aiter", True, "aiter"),
("per_row", False, "per_row"),
],
)
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
monkeypatch: pytest.MonkeyPatch,
configured: str,
full_cudagraph: bool,
expected: str,
) -> None:
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
use the context length to pick AITER and must stay on the in-tree kernel.
Explicit backends keep working, and outside FULL the long-context heuristic
still narrows to AITER."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.glm5next.amd import sparse_indexer
monkeypatch.setattr(
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
)
monkeypatch.setattr(
rocm_aiter_ops,
"is_indexer_top_k_supported",
lambda **kwargs: True,
raising=False,
)
assert (
sparse_indexer._kpool_decode_topk_backend(
configured,
num_rows=8,
max_valid_seq_len=64 * 1024,
select_k=512,
index_kpool=4,
full_cudagraph=full_cudagraph,
)
== expected
)
@@ -1,59 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm guards for shared Inkling model-definition contracts."""
from pathlib import Path
import torch
import vllm.models.inkling as inkling
from vllm.lora.utils import get_supported_lora_modules
from vllm.model_executor.models.interfaces import supports_lora
from vllm.models.inkling.amd import model as amd_model
from vllm.models.inkling.amd.mtp import InklingMTP
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
inkling_dir = Path(amd_model.__file__).parents[1]
for filename in ("logits_processor.py", "mtp.py"):
assert (inkling_dir / "amd" / filename).read_bytes() == (
inkling_dir / "nvidia" / filename
).read_bytes()
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
assert inkling.InklingMTP is InklingMTP
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
model_cls = amd_model._TmlForCausalLMBase
assert supports_lora(model_cls)
assert model_cls.packed_modules_mapping == {
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
"w13": ["w1", "w3"],
}
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
model = torch.nn.Module()
model.embedding_modules = model_cls.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
weight = torch.empty(1)
name, mapped_weight = next(
iter(
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
)
)
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
assert mapped_weight.shard_id == 0
assert amd_model._is_peft_adapter_weight(name)
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
assert lm_head_name == ["lm_head.lora_B.weight"]
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
from types import SimpleNamespace
import torch
from torch import nn
from vllm.models.inkling.amd.moe import InklingMoE
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
moe = InklingMoE.__new__(InklingMoE)
nn.Module.__init__(moe)
moe.n_routed_experts = 4
routed = SimpleNamespace(
moe_config=SimpleNamespace(
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
),
# Logical intermediate per rank is 3; AITER pads it to 4.
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
)
moe.experts = SimpleNamespace(routed_experts=routed)
moe._local_expert_slots = ( # type: ignore[method-assign]
lambda: dict(enumerate(range(4)))
)
return moe, routed
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
moe, routed = _fake_moe()
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
moe.load_expert_weight("experts.w13_weight", w13)
moe.load_expert_weight("experts.w2_weight", w2)
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
# one padding row in each destination half and one padding w2 column.
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
def test_unflattens_quark_scales_and_preserves_scale_padding():
moe, routed = _fake_moe()
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
w13_scale = flat_w13_scale.view(4, 12, 2)
w2_scale = flat_w2_scale.view(4, 5, 4)
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
@@ -1,425 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
import pytest
import torch
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
bucket_max_seqlen_q,
inkling_fa4_rel_attention,
use_gfx950_gluon_decode,
use_gfx950_gluon_extend,
)
from vllm.models.inkling.amd.ops.rel_attention_decode import (
decode_split_count,
use_split_kv_decode,
)
HEAD_DIM = 128
DTYPE = torch.bfloat16
def _reference(
q: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
rel_logits: torch.Tensor,
block_table: torch.Tensor,
q_lens: list[int],
kv_lens: list[int],
rel_extent: int,
window_left: int | None,
) -> torch.Tensor:
num_heads = q.shape[1]
num_kv_heads = key_cache.shape[2]
gqa_group = num_heads // num_kv_heads
out: list[torch.Tensor] = []
q_start = 0
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
page_size = key_cache.shape[1]
num_pages = (kv_len + page_size - 1) // page_size
page_ids = block_table[req, :num_pages].long()
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
k = k.float().repeat_interleave(gqa_group, dim=1)
v = v.float().repeat_interleave(gqa_group, dim=1)
q_req = q[q_start : q_start + q_len].float()
rel_req = rel_logits[q_start : q_start + q_len].float()
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
k_pos = torch.arange(kv_len, device=q.device)[None, :]
distance = q_pos - k_pos
rel_idx = distance.clamp(0, rel_extent - 1)
bias = rel_req.permute(1, 0, 2).gather(
2, rel_idx[None].expand(num_heads, -1, -1)
)
in_rel_extent = (distance >= 0) & (distance < rel_extent)
scores += torch.where(in_rel_extent[None], bias, 0.0)
masked = distance < 0
if window_left is not None:
masked |= distance > window_left
scores.masked_fill_(masked[None], float("-inf"))
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
q_start += q_len
return torch.cat(out).to(DTYPE)
def test_query_length_bucket():
assert bucket_max_seqlen_q(1) == 1
assert bucket_max_seqlen_q(17) == 32
assert bucket_max_seqlen_q(33) == 64
def test_split_kv_dispatch_thresholds():
assert not use_split_kv_decode(
max_query_len=4,
max_kv_len=65536,
page_size=128,
window_left=-1,
)
assert not use_split_kv_decode(
max_query_len=1,
max_kv_len=4096,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=8192,
page_size=16,
window_left=-1,
)
assert use_split_kv_decode(
max_query_len=1,
max_kv_len=512,
page_size=128,
window_left=511,
)
assert decode_split_count(512, 511) == 4
assert decode_split_count(65536, -1) == 32
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
assert use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=-1,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=128,
head_dim=128,
window_left=511,
)
assert not use_gfx950_gluon_extend(
max_query_len=4,
max_kv_len=8192,
page_size=16,
head_dim=128,
window_left=-1,
)
@pytest.mark.parametrize(
("page_size", "window_left", "max_kv_len"),
[
(16, None, None),
(16, 15, None),
(128, None, 8192),
],
)
@torch.inference_mode()
def test_ragged_multi_page_relative_attention(
page_size: int,
window_left: int | None,
max_kv_len: int | None,
):
"""Covers full/chunked prefill, decode, GQA, and the local window."""
torch.manual_seed(19 + int(window_left is not None))
device = "cuda"
q_lens = [17, 1]
kv_lens = [35, 80]
num_heads = 8
num_kv_heads = 2
rel_extent = 16
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
1 + len(q_lens) * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
)
block_table = torch.stack(
[
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
for req in range(len(q_lens))
]
).to(device=device, dtype=torch.int32)
cu_seqlens_q = torch.tensor(
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
device=device,
dtype=torch.int32,
)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
window_size = (-1, -1) if window_left is None else (window_left, 0)
preallocated = torch.empty_like(q)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
out=preallocated,
)
assert actual.data_ptr() == preallocated.data_ptr()
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
[
(16, [257, 513], 1, 64, None),
(128, [257, 512], 2, 512, 511),
],
)
@torch.inference_mode()
def test_split_kv_decode_matches_reference(
page_size: int,
kv_lens: list[int],
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
):
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
torch.manual_seed(41 + page_size)
device = "cuda"
batch_size = len(kv_lens)
q_lens = [1] * batch_size
num_heads = 8
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
key_cache = torch.randn(
batch_size * max_pages,
page_size,
num_kv_heads,
HEAD_DIM,
device=device,
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn_like(key_cache)
rel_logits = torch.randn(
batch_size,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * max_pages,
device=device,
dtype=torch.int32,
).view(batch_size, max_pages)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
batch_size + 1,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=1,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=max_kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@pytest.mark.parametrize(
(
"q_len",
"kv_len",
"num_kv_heads",
"rel_extent",
"window_left",
"gluon_enabled",
),
[
(1, 513, 2, 512, 511, True),
(1, 513, 2, 512, 511, False),
(4, 8192, 1, 1024, None, True),
],
)
@torch.inference_mode()
def test_gfx950_page128_packed_kv_views_match_reference(
monkeypatch: pytest.MonkeyPatch,
q_len: int,
kv_len: int,
num_kv_heads: int,
rel_extent: int,
window_left: int | None,
gluon_enabled: bool,
):
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
monkeypatch.setenv(
"INKLING_GFX950_GLUON",
"1" if gluon_enabled else "0",
)
torch.manual_seed(71 + q_len)
device = "cuda"
page_size = 128
batch_size = 2
num_heads = 8
q_lens = [q_len] * batch_size
kv_lens = [kv_len] * batch_size
pages_per_req = (kv_len + page_size - 1) // page_size
q = torch.randn(
batch_size * q_len,
num_heads,
HEAD_DIM,
device=device,
dtype=DTYPE,
)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
# producing non-contiguous views whose page/head strides include both.
packed_kv = torch.randn(
batch_size * pages_per_req,
num_kv_heads,
page_size,
2 * HEAD_DIM,
device=device,
dtype=DTYPE,
)
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
assert not key_cache.is_contiguous()
assert key_cache.stride() == value_cache.stride()
rel_logits = torch.randn(
batch_size * q_len,
num_heads,
rel_extent,
device=device,
dtype=DTYPE,
)
block_table = torch.arange(
batch_size * pages_per_req,
device=device,
dtype=torch.int32,
).view(batch_size, pages_per_req)
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
cu_seqlens_q = torch.arange(
0,
batch_size * q_len + 1,
q_len,
device=device,
dtype=torch.int32,
)
window_size = (-1, -1) if window_left is None else (window_left, 0)
actual = inkling_fa4_rel_attention(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=q_len,
softmax_scale=1.0 / HEAD_DIM,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
max_kv_len=kv_len,
)
expected = _reference(
q,
key_cache,
value_cache,
rel_logits,
block_table,
q_lens,
kv_lens,
rel_extent,
window_left,
)
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
@@ -1,116 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
chained backbone embed_norm and the depth embed_norm), and the fused row
gather matches ``F.embedding``.
"""
from typing import cast
import pytest
import torch
from vllm.models.inkling.amd.ops.norm import (
embed_dual_rmsnorm_cat,
embed_rmsnorm,
rmsnorm,
)
EPS = 1e-6
VOCAB = 4096
BF16_ATOL = torch.finfo(torch.bfloat16).eps
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
# Different Triton reduction schedules can move a small number of outputs
# by one bf16 epsilon while remaining numerically equivalent.
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
def _ref(hidden, w_h, w_e, emb, w_pre=None):
if w_pre is not None:
emb = rmsnorm(emb, w_pre, EPS)
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
emb = table[ids.long()]
# Fused gather + chained backbone pre-norm (the decode draft-step path).
out = embed_dual_rmsnorm_cat(
hidden,
w_h,
w_e,
EPS,
input_ids=ids,
embed_table=table,
pre_norm_weight=w_pre,
)
assert out.shape == (t, 2 * n)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
# embeddings, already backbone-normed).
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
# Fused gather, no pre-norm (use_embed_norm=False).
out = embed_dual_rmsnorm_cat(
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
)
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
ref_emb = table[ids.long()]
# Gather + embed_norm (base model / MTP prefill embed path).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
assert out.shape == (t, n)
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
# Pure gather (use_embed_norm=False / replicated module forward).
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
assert torch.equal(out, ref_emb)
# Chained first-layer attn_norm (the target text-path forward): one launch
# emits both the residual and layer 0's normed attention input.
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
)
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
_assert_bf16_close(res, ref_res)
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
# Chained without embed_norm (use_embed_norm=False).
res, attn_in = cast(
tuple[torch.Tensor, torch.Tensor],
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
)
assert torch.equal(res, ref_emb)
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
@@ -1,20 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
def test_runtime_sconv_block_size_tracks_unified_cache_page():
"""The cache planner may enlarge W=4 blocks to match attention pages."""
owner = InklingConvState.__new__(InklingConvState)
nn.Module.__init__(owner)
owner.block_size = 4
owner.kv_cache = torch.tensor([])
assert owner.cache_block_size == 4
owner.kv_cache = torch.empty(2, 1, 32, 1024)
assert owner.cache_block_size == 32
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.models.inkling.common.mm_preprocess import InklingMultiModalDataParser
from vllm.models.inkling.common.towers import plan_out_scales
from vllm.models.inkling.configs import (
InklingAudioConfig,
InklingModelConfig,
InklingVisionConfig,
)
from vllm.models.inkling.nvidia.sconv_swa_attn import (
InklingSconvMetadataBuilder,
)
from vllm.v1.attention.backend import AttentionCGSupport
def test_vision_scale_plan_matches_released_config():
assert plan_out_scales(2, 40, 4) == [
(1, 1, 1, 3),
(1, 5, 5, 128),
(1, 10, 10, 320),
(1, 40, 40, 4800),
(2, 40, 40, 9600),
]
def test_vision_scale_plan_breaks_assignment_ties_in_order():
reductions = [np.prod(scale[:-1]) for scale in plan_out_scales(2, 52, 4)]
assert reductions == sorted(set(reductions))
@pytest.mark.parametrize(
("config_cls", "kwargs", "missing"),
[
(InklingAudioConfig, {"decoder_dmodel": 16}, "n_mel_bins"),
(InklingVisionConfig, {"decoder_dmodel": 16}, "vision_encoder_type"),
],
)
def test_enabled_tower_requires_architecture_fields(config_cls, kwargs, missing):
with pytest.raises(ValueError, match=missing):
config_cls(**kwargs)
def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
parser = InklingMultiModalDataParser(target_sr=16_000, target_channels=1)
with pytest.raises(ValueError, match="ambiguous channel layout"):
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
def test_inkling_mtp_chain_norm_is_disabled_by_default():
assert InklingModelConfig().chain_hidden_post_norm is False
def test_inkling_supports_piecewise_cudagraphs():
support = InklingSconvMetadataBuilder.get_cudagraph_support
assert support(None, None) == AttentionCGSupport.UNIFORM_BATCH
compilation_config = CompilationConfig(
cudagraph_mode=CUDAGraphMode.PIECEWISE,
splitting_ops=[],
)
resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
AttentionCGSupport.UNIFORM_BATCH,
"InklingSconvBackend",
)
assert resolved_mode == CUDAGraphMode.PIECEWISE
@@ -1,428 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness test for the Inkling FA4 relative-attention score-mod kernel.
Checks ``_INKLING_FA4_REL_ATTENTION_KERNEL`` against a pure-PyTorch reference
that implements the relative bias exactly as documented in the Inkling architecture
guide::
logit(i, j, h) = (1 / head_dim) * dot(q[i, h], k[j, h]) + rel_bias(i, j, h)
rel_bias(i, j, h) = rel_logits[i, h, i - j] if 0 <= i - j < rel_extent
= 0 otherwise
with causal (and optionally sliding-window) masking handled by the backend.
"""
import importlib
import pytest
import torch
from vllm.models.inkling.nvidia.attention import (
InklingAttention,
compute_log_scaling_tau,
)
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
_INKLING_FA4_REL_ATTENTION_KERNEL,
_use_sheared_bias,
inkling_fa4_num_splits,
)
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
NUM_HEADS = [(4, 4), (8, 2)] # (num_heads, num_kv_heads)
GLOBAL_REL_EXTENTS = [128, 1024]
LOCAL_REL_EXTENTS = [128, 256]
HEAD_DIM = 128
BLOCK_SIZE = 16
DTYPE = torch.bfloat16
def test_log_scaling_tau_matches_reference():
positions = torch.tensor([0, 127999, 128000, 999999], dtype=torch.int64)
actual = compute_log_scaling_tau(positions, 128000, 0.1)
expected = 1.0 + 0.1 * torch.log(
torch.clamp((positions + 1).float() / 128000.0, min=1.0)
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
def test_split_packed_kv_cache():
attention = InklingAttention.__new__(InklingAttention)
torch.nn.Module.__init__(attention)
attention.head_dim = 8
attention.kv_cache = torch.arange(2 * 3 * 4 * 16).reshape(2, 3, 4, 16)
key_cache, value_cache = attention._split_kv_cache()
assert key_cache.shape == value_cache.shape == (2, 4, 3, 8)
torch.testing.assert_close(key_cache, attention.kv_cache[..., :8].transpose(1, 2))
torch.testing.assert_close(value_cache, attention.kv_cache[..., 8:].transpose(1, 2))
def test_num_splits_hopper_is_unsplit(monkeypatch):
monkeypatch.setattr(
current_platform,
"get_device_capability",
lambda: DeviceCapability(major=9, minor=0),
)
assert (
inkling_fa4_num_splits(
is_local=False,
batch_size=1,
max_query_len=1,
num_heads=16,
num_kv_heads=2,
max_kv_len=1_048_576,
)
== 1
)
@pytest.mark.parametrize(
("major", "expected"),
[(9, False), (10, True), (11, True), (12, False)],
)
def test_sheared_bias_architecture_selection(monkeypatch, major, expected):
monkeypatch.setattr(
current_platform,
"get_device_capability",
lambda: DeviceCapability(major=major, minor=0),
)
_use_sheared_bias.cache_clear()
try:
assert _use_sheared_bias() is expected
finally:
_use_sheared_bias.cache_clear()
@pytest.fixture
def blackwell_platform(monkeypatch):
monkeypatch.setattr(
current_platform,
"get_device_capability",
lambda: DeviceCapability(major=10, minor=0),
)
@pytest.mark.parametrize(
("batch_size", "max_query_len", "expected"),
[
(1, 1, (16, 32, 128, 128)),
(8, 1, (2, 4, 8, 16)),
(32, 1, (1, 1, 2, 4)),
(1, 128, (2, 4, 8, 16)),
(1, 2048, (1, 1, 1, 1)),
],
)
def test_num_splits_all_tp(blackwell_platform, batch_size, max_query_len, expected):
actual = tuple(
inkling_fa4_num_splits(
is_local=False,
batch_size=batch_size,
max_query_len=max_query_len,
num_heads=64 // tp,
num_kv_heads=8 // tp,
max_kv_len=131072,
)
for tp in (1, 2, 4, 8)
)
assert actual == expected
@pytest.mark.parametrize("tp", [1, 2, 4, 8])
def test_num_splits_local_is_unsplit(tp):
assert (
inkling_fa4_num_splits(
is_local=True,
batch_size=1,
max_query_len=1,
num_heads=64 // tp,
num_kv_heads=16 // tp,
max_kv_len=512,
)
== 1
)
@pytest.mark.parametrize(
("max_kv_len", "expected"),
[(8192, 32), (65536, 64), (1048576, 128)],
)
@pytest.mark.parametrize("tp", [4, 8])
def test_num_splits_long_context_bound(blackwell_platform, tp, max_kv_len, expected):
assert (
inkling_fa4_num_splits(
is_local=False,
batch_size=1,
max_query_len=1,
num_heads=64 // tp,
num_kv_heads=8 // tp,
max_kv_len=max_kv_len,
)
== expected
)
def _ref_rel_attn(
q: torch.Tensor, # [total_q, H, D]
key_cache: torch.Tensor, # [num_blocks, block, Hkv, D]
value_cache: torch.Tensor,
rel_logits: torch.Tensor, # [total_q, H, rel_extent]
*,
q_lens: list[int],
kv_lens: list[int],
block_table: torch.Tensor,
scale: float,
rel_extent: int,
window_left: int | None,
) -> torch.Tensor:
num_kv_heads = key_cache.shape[2]
num_heads = q.shape[1]
g = num_heads // num_kv_heads
bt = block_table.cpu().numpy()
out = torch.empty_like(q)
start = 0
for i, (ql, kl) in enumerate(zip(q_lens, kv_lens)):
qi = q[start : start + ql].float() # [ql, H, D]
rl = rel_logits[start : start + ql].float() # [ql, H, rel_extent]
nblk = (kl + BLOCK_SIZE - 1) // BLOCK_SIZE
blk = bt[i, :nblk]
k = key_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
v = value_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
k = k.repeat_interleave(g, dim=1) # [kl, H, D]
v = v.repeat_interleave(g, dim=1)
# [H, ql, kl]
scores = torch.einsum("qhd,khd->hqk", qi, k) * scale
dev = q.device
qpos = torch.arange(ql, device=dev).view(ql, 1) + (kl - ql) # query pos
kpos = torch.arange(kl, device=dev).view(1, kl)
dist = qpos - kpos # [ql, kl] = i - j
# Relative bias: rel_logits[i, h, dist] when 0 <= dist < rel_extent.
in_rng = (dist >= 0) & (dist < rel_extent) # [ql, kl]
idx = dist.clamp(0, rel_extent - 1)
# gather per head: bias[h, i, j] = rl[i, h, idx[i, j]]
bias = rl.permute(1, 0, 2).gather( # [H, ql, rel_extent]
2, idx.unsqueeze(0).expand(num_heads, -1, -1)
) # [H, ql, kl]
bias = torch.where(in_rng.unsqueeze(0), bias, torch.zeros_like(bias))
scores = scores + bias
mask = dist < 0 # causal
if window_left is not None:
mask = mask | (dist > window_left)
scores.masked_fill_(mask.unsqueeze(0), float("-inf"))
probs = torch.softmax(scores, dim=-1)
out[start : start + ql] = torch.einsum("hqk,khd->qhd", probs, v).to(q.dtype)
start += ql
return out
def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0):
torch.manual_seed(seed)
device = "cuda"
q_lens = [s[0] for s in seq_lens]
kv_lens = [s[1] for s in seq_lens]
total_q = sum(q_lens)
num_seqs = len(seq_lens)
scale = 1.0 / HEAD_DIM
# q/k are RMS-normed in the model (unit-ish norm); normalize here so the
# logit magnitudes are realistic and the bias is not numerically dwarfed.
q = torch.randn(total_q, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
# Paged KV cache.
max_blocks = (max(kv_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE
num_blocks = num_seqs * max_blocks + 1
key_cache = torch.randn(
num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
)
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
value_cache = torch.randn(
num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
)
# Distinct blocks per sequence (block 0 left as a never-referenced pad).
block_table = torch.zeros(num_seqs, max_blocks, dtype=torch.int32, device=device)
for i in range(num_seqs):
block_table[i] = torch.arange(
1 + i * max_blocks, 1 + (i + 1) * max_blocks, dtype=torch.int32
)
cu_seqlens_q = torch.tensor(
[0, *torch.cumsum(torch.tensor(q_lens), 0).tolist()],
dtype=torch.int32,
device=device,
)
cache_seqlens = torch.tensor(kv_lens, dtype=torch.int32, device=device)
rel_logits = torch.randn(total_q, num_heads, rel_extent, device=device, dtype=DTYPE)
window_size = (-1, -1) if window_left is None else (window_left, 0)
preallocated_out = torch.empty_like(q)
num_splits = inkling_fa4_num_splits(
is_local=window_left is not None,
batch_size=num_seqs,
max_query_len=max(q_lens),
num_heads=num_heads,
num_kv_heads=num_kv_heads,
max_kv_len=max(kv_lens),
)
out = _INKLING_FA4_REL_ATTENTION_KERNEL(
q,
key_cache,
value_cache,
block_table=block_table,
cache_seqlens=cache_seqlens,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=max(q_lens),
softmax_scale=scale,
causal=True,
window_size=window_size,
rel_extent=rel_extent,
rel_logits=rel_logits,
num_splits=num_splits,
out=preallocated_out,
)
assert out.data_ptr() == preallocated_out.data_ptr()
out = out.view(total_q, num_heads, HEAD_DIM)
ref = _ref_rel_attn(
q,
key_cache,
value_cache,
rel_logits,
q_lens=q_lens,
kv_lens=kv_lens,
block_table=block_table,
scale=scale,
rel_extent=rel_extent,
window_left=window_left,
)
torch.testing.assert_close(out.float(), ref.float(), atol=2e-2, rtol=2e-2)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.skipif(
_cap is None or _cap.major < 9,
reason="FA4 score-mod requires Hopper+ (SM90+)",
)
@torch.inference_mode()
def test_score_mod_relative_attention(monkeypatch):
module = importlib.import_module("vllm.models.inkling.nvidia.ops.fa4_rel_attention")
monkeypatch.setattr(module, "_use_sheared_bias", lambda: False)
_run_case(
[(64, 64), (1, 80)],
num_heads=4,
num_kv_heads=4,
rel_extent=128,
window_left=None,
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.skipif(
_cap is None or _cap.major < 9,
reason="FA4 score-mod requires Hopper+ (SM90+)",
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize(
"seq_lens",
[
[(64, 64)], # single full prefill
[(64, 64), (33, 33), (17, 17)], # ragged prefill batch
[(512, 512)], # seq_len >> rel_extent (most keys get zero bias)
[(300, 300), (512, 512), (129, 129)], # large ragged batch
],
)
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
@torch.inference_mode()
def test_full_attention(seq_lens, num_heads, rel_extent):
# rel_extent=128 exercises the out-of-range (zero bias) path; 1024 covers all.
# With the 512-token cases and rel_extent=128, query/seq lengths are far
# larger than rel_extent so the vast majority of (i, j) pairs are out of
# range and must contribute zero bias.
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.skipif(
_cap is None or _cap.major < 9,
reason="FA4 score-mod requires Hopper+ (SM90+)",
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize(
"seq_lens",
[
[(200, 512)], # chunked prefill: q_len=200 (> rel_extent), 312 cached
[(200, 512), (50, 300), (1, 400)], # mixed chunked + decode
],
)
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
@torch.inference_mode()
def test_chunked_prefill(seq_lens, num_heads, rel_extent):
# q_len < kv_len with q_len itself larger than rel_extent (for the 128 case):
# exercises the seqlen_k - seqlen_q offset together with the out-of-range path.
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.skipif(
_cap is None or _cap.major < 9,
reason="FA4 score-mod requires Hopper+ (SM90+)",
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize(
"seq_lens",
[
[(64, 64), (40, 40)], # seq_len > window
[(512, 512), (300, 300)], # seq_len/query_len >> window
[(1, 512)], # decode with kv_len >> window
],
)
@pytest.mark.parametrize("local_extent", LOCAL_REL_EXTENTS)
@torch.inference_mode()
def test_sliding_window(seq_lens, num_heads, local_extent):
# Local layers use window_size=(local_extent-1, 0) and rel_extent==local_extent.
# With the 512-token cases, query/seq lengths far exceed the window so most
# keys are masked out by the sliding window.
_run_case(
seq_lens,
num_heads[0],
num_heads[1],
rel_extent=local_extent,
window_left=local_extent - 1,
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.skipif(
_cap is None or _cap.major < 9,
reason="FA4 score-mod requires Hopper+ (SM90+)",
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize(
"seq_lens",
[
[(1, 50)],
[(1, 50), (1, 7), (1, 200)],
[(1, 512), (1, 333)], # kv_len >> rel_extent
],
)
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
@torch.inference_mode()
def test_decode(seq_lens, num_heads, rel_extent):
# q_len=1 with kv_len>q_len: the score-mod's seqlen_k - seqlen_q offset path.
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
-124
View File
@@ -1,124 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import torch
import vllm.models.inkling.nvidia.ops.fa4_rel_attention as fa4_rel_attention
from vllm.models.inkling.configs import InklingMMConfig, InklingModelConfig
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
InklingFA4RelAttentionKernel,
_num_warps_bucket,
bucket_max_seqlen_q,
inkling_fa4_num_splits,
)
def _vllm_config_from_reference_config(config: SimpleNamespace) -> SimpleNamespace:
return SimpleNamespace(
model_config=SimpleNamespace(
hf_config=InklingMMConfig(
text_config=InklingModelConfig(
num_attention_heads=config.num_heads,
num_key_value_heads=config.num_kv_heads,
head_dim=config.head_dim,
rel_extent=config.rel_extent,
swa_num_attention_heads=config.num_heads,
swa_num_key_value_heads=config.num_kv_heads,
swa_head_dim=config.head_dim,
sliding_window_size=config.rel_extent,
)
),
max_model_len=config.max_kv_len,
dtype=config.dtype,
),
cache_config=SimpleNamespace(
cache_dtype="auto",
block_size=config.block_size,
),
scheduler_config=SimpleNamespace(
max_num_seqs=config.max_num_reqs,
max_num_batched_tokens=config.max_num_batched_tokens,
),
)
def test_bucket_max_seqlen_q():
assert [bucket_max_seqlen_q(n) for n in range(1, 10)] == [
1,
2,
4,
4,
8,
8,
8,
8,
16,
]
def test_warmup_enumerates_every_runtime_compile_class(monkeypatch):
monkeypatch.setattr(
fa4_rel_attention,
"get_tensor_model_parallel_world_size",
lambda: 1,
)
config = SimpleNamespace(
num_heads=16,
num_kv_heads=2,
head_dim=128,
rel_extent=1024,
window_size=(-1, -1),
is_local=False,
max_kv_len=65536,
dtype=torch.bfloat16,
kv_dtype=torch.bfloat16,
block_size=16,
max_num_reqs=64,
max_num_batched_tokens=192,
)
vllm_config = _vllm_config_from_reference_config(config)
kernel = InklingFA4RelAttentionKernel()
warmed_keys = set(kernel.get_warmup_keys(vllm_config))
expected_keys: set[InklingFA4RelAttentionKernel.CompileKey] = set()
for is_local in (False, True):
max_kv_len = config.rel_extent if is_local else config.max_kv_len
window_size = (config.rel_extent - 1, 0) if is_local else (-1, -1)
for query_len in range(1, config.max_num_batched_tokens + 1):
max_seqlen_q = bucket_max_seqlen_q(query_len)
max_num_reqs = min(
config.max_num_reqs,
config.max_num_batched_tokens - query_len + 1,
)
for num_reqs in range(1, max_num_reqs + 1):
num_splits = inkling_fa4_num_splits(
is_local=is_local,
batch_size=num_reqs,
max_query_len=max_seqlen_q,
num_heads=config.num_heads,
num_kv_heads=config.num_kv_heads,
max_kv_len=max_kv_len,
)
expected_keys.add(
InklingFA4RelAttentionKernel.CompileKey(
is_local=is_local,
num_heads=config.num_heads,
num_kv_heads=config.num_kv_heads,
head_dim=config.head_dim,
rel_extent=config.rel_extent,
dtype=config.dtype,
kv_dtype=config.kv_dtype,
block_size=config.block_size,
window_size=window_size,
max_seqlen_q=max_seqlen_q,
num_splits=num_splits,
num_warps_bucket=(
_num_warps_bucket(num_reqs) if num_splits > 1 else None
),
large_num_reqs=num_reqs > 1024,
)
)
assert warmed_keys == expected_keys
-138
View File
@@ -1,138 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused Inkling vision/audio tower kernels.
Each kernel is checked against a pure-PyTorch reference implementing the
towers' original op sequence (native ``rms_norm`` semantics: fp32 variance,
bf16-rounded weight multiply; exact-erf GELU; fp32-accumulated embedding sum).
The fused kernels keep the same accumulation dtype and rounding points, so
outputs differ from the reference only by reduction order — a few bf16 ulps.
"""
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("requires CUDA", allow_module_level=True)
from vllm.models.inkling.common.towers import fold_timespace_to_depth
from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm, rmsnorm_gelu
DTYPE = torch.bfloat16
def _bf16_spacing(x: torch.Tensor) -> torch.Tensor:
return torch.exp2(torch.floor(torch.log2(x.float().abs().clamp(min=1e-30)))) * (
2**-7
)
def _assert_close_ulps(
got: torch.Tensor,
ref: torch.Tensor,
max_ulps: float = 4.0,
atol: float = 1e-3,
pre_act: torch.Tensor | None = None,
) -> None:
"""Fused vs reference differ only by fp32 reduction order, which shows up
as a few bf16 ulps at any magnitude — a fixed rtol misrepresents that, so
compare against the reference's local ulp spacing. ``pre_act`` (the
reference pre-activation) adds derivative-propagated slack: near
``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
g, r = got.float(), ref.float()
tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
if pre_act is not None:
tol = tol + 2.5 * _bf16_spacing(pre_act)
bad = (g - r).abs() > tol
assert not bad.any(), (
f"{int(bad.sum())}/{ref.numel()} elements beyond tolerance; "
f"max abs diff {(g - r).abs().max().item():.3e}"
)
def _ref_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
# Matches ir.ops.rms_norm: fp32 normalize, bf16 weight multiply.
x32 = x.float()
var = x32.pow(2).mean(dim=-1, keepdim=True)
xn = x32 * torch.rsqrt(var + eps)
return (xn.to(weight.dtype) * weight).to(x.dtype)
@pytest.mark.parametrize("rows", [1, 7, 1200, 38400])
@pytest.mark.parametrize("dim", [128, 320, 4800, 6144])
@pytest.mark.parametrize("gelu", [True, False])
def test_rmsnorm_gelu(rows: int, dim: int, gelu: bool) -> None:
torch.manual_seed(rows * dim)
x = torch.randn(rows, dim, device="cuda", dtype=DTYPE)
w = torch.randn(dim, device="cuda", dtype=DTYPE)
h = _ref_rms_norm(x, w, 1e-5)
ref = F.gelu(h) if gelu else h
got = rmsnorm_gelu(x, w, 1e-5, gelu=gelu)
_assert_close_ulps(got, ref, pre_act=h if gelu else None)
@pytest.mark.parametrize("n", [1, 5, 64])
@pytest.mark.parametrize(
"shape,fold",
[
((2, 8, 8, 128), (1, 2)), # the real L0 -> L1 vision transition
((2, 4, 4, 320), (1, 2)),
((2, 8, 8, 128), (2, 2)), # temporal + spatial fold
],
)
def test_rmsnorm_gelu_folded_store(
n: int, shape: tuple[int, ...], fold: tuple[int, int]
) -> None:
torch.manual_seed(n)
x = torch.randn(n, *shape, device="cuda", dtype=DTYPE)
w = torch.randn(shape[-1], device="cuda", dtype=DTYPE)
plain = rmsnorm_gelu(x, w, 1e-5, gelu=True)
ref = fold_timespace_to_depth(plain, *fold)
got = rmsnorm_gelu(x, w, 1e-5, gelu=True, fold=fold)
# The folded store is a pure permutation of the unfolded output.
assert got.shape == ref.shape
torch.testing.assert_close(got, ref, rtol=0, atol=0)
@pytest.mark.parametrize("num_frames", [1, 3, 100, 4097])
@pytest.mark.parametrize("with_norm", [True, False])
def test_dmel_embed_sum_norm(num_frames: int, with_norm: bool) -> None:
torch.manual_seed(num_frames)
n_bins, vocab, dim = 80, 16, 6144
idx = torch.randint(
0, vocab, (num_frames, n_bins), device="cuda", dtype=torch.int32
)
table = torch.randn(n_bins * vocab, dim, device="cuda", dtype=DTYPE)
norm_w = torch.randn(dim, device="cuda", dtype=DTYPE)
flat = (torch.arange(n_bins, device="cuda", dtype=torch.int32) * vocab).unsqueeze(
0
) + idx
ref = (
F.embedding(flat.reshape(-1).long(), table)
.reshape(num_frames, n_bins, dim)
.sum(dim=1)
)
if with_norm:
ref = _ref_rms_norm(ref, norm_w, 1e-6)
got = dmel_embed_sum_norm(idx, table, norm_w if with_norm else None, 1e-6)
_assert_close_ulps(got, ref)
def test_empty_inputs() -> None:
x = torch.empty(0, 320, device="cuda", dtype=DTYPE)
w = torch.randn(320, device="cuda", dtype=DTYPE)
assert rmsnorm_gelu(x, w, 1e-5).shape == (0, 320)
idx = torch.empty(0, 80, device="cuda", dtype=torch.int32)
table = torch.randn(1280, 6144, device="cuda", dtype=DTYPE)
assert dmel_embed_sum_norm(idx, table, None, 0.0).shape == (0, 6144)
@@ -1,249 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.lora.utils import get_supported_lora_modules
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
from vllm.models.inkling.nvidia import moe
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
from vllm.platforms import current_platform
def test_gate_loads_directly_into_padded_runtime_weight() -> None:
gate = moe.InklingGate(
d_model=4,
n_routed_experts=5,
n_shared_experts=2,
experts_per_token=2,
route_scale=1.0,
)
loaded = torch.arange(28, dtype=gate.weight.dtype).reshape(7, 4)
gate.weight.weight_loader(gate.weight, loaded)
assert gate.weight.shape == (8, 4)
torch.testing.assert_close(gate.weight[:7], loaded)
torch.testing.assert_close(gate.weight[7], torch.zeros(4))
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
@pytest.mark.parametrize(("num_tokens", "expected_calls"), [(1, 1), (64, 1), (65, 0)])
def test_gate_uses_ll_bf16_gemm_through_token_limit(
monkeypatch, num_tokens, expected_calls
) -> None:
gate = moe.InklingGate(
d_model=8,
n_routed_experts=5,
n_shared_experts=2,
experts_per_token=2,
route_scale=1.0,
).to(device="cuda", dtype=torch.bfloat16)
hidden_states = torch.randn(num_tokens, 8, device="cuda", dtype=torch.bfloat16)
calls = []
def fake_ll_bf16_gemm(x, weight):
calls.append((x, weight))
return torch.ones(num_tokens, 8, device="cuda", dtype=torch.float32)
monkeypatch.setattr(
moe.current_platform, "has_device_capability", lambda capability: True
)
monkeypatch.setattr(moe.ll_bf16, "is_available", lambda: True)
monkeypatch.setattr(moe.ll_bf16, "ll_bf16_gemm", fake_ll_bf16_gemm)
logits = gate.compute_logits(hidden_states)
assert len(calls) == expected_calls
if calls:
assert calls[0][0] is hidden_states
assert calls[0][1] is gate.weight
assert logits.shape == (num_tokens, 8)
assert logits.dtype == torch.float32
def test_gate_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.gate = moe.InklingGate(
d_model=4,
n_routed_experts=5,
n_shared_experts=2,
experts_per_token=2,
route_scale=1.0,
)
assert "gate" not in get_supported_lora_modules(model)
def test_custom_embedding_is_not_a_lora_target() -> None:
model = torch.nn.Module()
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
supported = get_supported_lora_modules(model)
assert "embed_tokens" not in supported
assert "lm_head" in supported
def test_inkling_mapper_maps_modelopt_exclusions() -> None:
quant_config = ModelOptNvFp4Config.from_config(
{
"quantization": {
"quant_algo": "NVFP4",
"group_size": 16,
"kv_cache_quant_algo": None,
"exclude_modules": [
"model.llm.layers.2.mlp.experts",
"model.llm.layers.2.mlp.shared_experts",
],
}
}
)
quant_config.apply_vllm_mapper(
_TmlForCausalLMBase.hf_to_vllm_mapper.get_rename_mapper()
)
assert quant_config.is_layer_excluded("model.layers.2.mlp.experts")
assert quant_config.is_layer_excluded("model.layers.2.mlp.shared_experts")
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
@pytest.mark.parametrize("projection", ["w13", "w2"])
@pytest.mark.parametrize("nested", [False, True])
@pytest.mark.parametrize(
"suffix",
[
"input_global_scale",
"weight_global_scale",
"weight_packed",
"weight_scale",
],
)
def test_inkling_mapper_maps_compressed_tensors_expert_params(
projection: str, suffix: str, nested: bool
) -> None:
projection_param = (
f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}"
)
source = f"model.llm.layers.2.mlp.experts.{projection_param}"
mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source])
assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"]
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
experts = SimpleNamespace(
w13_input_scale=torch.nn.Parameter(torch.empty(3, 2)),
w2_input_scale=torch.nn.Parameter(torch.empty(3)),
)
layer = SimpleNamespace(experts=SimpleNamespace(routed_experts=experts))
loaded = moe.InklingMoE.load_expert_weight(
layer,
f"experts.{projection}_weight.input_amax",
torch.tensor([amax]),
)
scale = getattr(experts, f"{projection}_input_scale")
expected = torch.full_like(scale, amax / (448.0 * 6.0))
torch.testing.assert_close(scale, expected)
assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
@pytest.mark.parametrize("projection", ["w13", "w2"])
@pytest.mark.parametrize("scale_kind", ["input", "weight"])
def test_moe_loads_compressed_tensors_global_scale(
projection: str, scale_kind: str
) -> None:
param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1))
if projection == "w2":
param = torch.nn.Parameter(param.squeeze(1))
experts = SimpleNamespace(
**{f"{projection}_{scale_kind}_global_scale": param},
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)),
)
layer = SimpleNamespace(
experts=SimpleNamespace(routed_experts=experts),
_local_expert_slots=lambda: {0: 0, 1: 1, 2: 2},
)
checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]])
loaded = moe.InklingMoE.load_expert_weight(
layer,
f"experts.{projection}_{scale_kind}_global_scale",
checkpoint_scale,
)
expected = (
checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0]
)
torch.testing.assert_close(param, expected)
assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"]
@pytest.mark.parametrize(("projection", "checkpoint_rows"), [("w13", 8), ("w2", 4)])
def test_moe_loads_channelwise_scale_for_tp(
projection: str, checkpoint_rows: int
) -> None:
param = torch.nn.Parameter(torch.empty(2, 4, 1))
experts = SimpleNamespace(
**{f"{projection}_weight_scale": param},
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=1)),
)
layer = SimpleNamespace(
experts=SimpleNamespace(routed_experts=experts),
_local_expert_slots=lambda: {0: 0, 2: 1},
)
checkpoint_scale = torch.arange(3 * checkpoint_rows).reshape(3, checkpoint_rows, 1)
loaded = moe.InklingMoE.load_expert_weight(
layer, f"experts.{projection}_weight_scale", checkpoint_scale
)
expected = checkpoint_scale[[0, 2]]
if projection == "w13":
expected = expected[:, 4:].reshape(2, 2, 2, 1).transpose(1, 2).flatten(1, 2)
torch.testing.assert_close(param, expected.float())
assert loaded == [f"experts.routed_experts.{projection}_weight_scale"]
def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8)
loaded = torch.arange(48, dtype=sink.w2_weight.dtype).reshape(2, 3, 8)
sink.load_weight("w2_weight", loaded)
expected = loaded[:, :, 4:].permute(1, 0, 2).reshape(3, 8)
torch.testing.assert_close(sink.w2_weight, expected)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
def test_sink_packed_weight_forward_matches_expert_sum(monkeypatch) -> None:
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8).to(
device="cuda", dtype=torch.bfloat16
)
torch.manual_seed(1)
w13 = torch.randn(2, 16, 3, dtype=torch.bfloat16)
w2 = torch.randn(2, 3, 8, dtype=torch.bfloat16)
sink.load_weight("w13_weight", w13)
sink.load_weight("w2_weight", w2)
x = torch.randn(5, 3, device="cuda", dtype=torch.bfloat16)
gammas = torch.randn(5, 2, device="cuda")
output = sink(x, gammas)
raw = torch.einsum("td,efd->tef", x, w13[:, 8:].to("cuda"))
hidden = torch.nn.functional.silu(raw[:, :, 0::2].float())
hidden = (hidden * raw[:, :, 1::2] * gammas[:, :, None]).to(torch.bfloat16)
expected = torch.einsum("tef,edf->td", hidden, w2[:, :, 4:].to("cuda"))
torch.testing.assert_close(output, expected, rtol=0, atol=0)
@@ -1,106 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bit-exactness tests for the fused MTP depth-layer input kernel.
``embed_dual_rmsnorm_cat`` must match the unfused module sequence exactly:
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
chained backbone embed_norm and the depth embed_norm), and the fused row
gather matches ``F.embedding``.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cuda():
pytest.skip("requires CUDA", allow_module_level=True)
from vllm.models.inkling.nvidia.ops.norm import (
embed_dual_rmsnorm_cat,
embed_rmsnorm,
rmsnorm,
)
EPS = 1e-6
VOCAB = 4096
def _ref(hidden, w_h, w_e, emb, w_pre=None):
if w_pre is not None:
emb = rmsnorm(emb, w_pre, EPS)
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
emb = table[ids.long()]
# Fused gather + chained backbone pre-norm (the decode draft-step path).
out = embed_dual_rmsnorm_cat(
hidden,
w_h,
w_e,
EPS,
input_ids=ids,
embed_table=table,
pre_norm_weight=w_pre,
)
assert out.shape == (t, 2 * n)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb, w_pre))
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
# embeddings, already backbone-normed).
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
# Fused gather, no pre-norm (use_embed_norm=False).
out = embed_dual_rmsnorm_cat(
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
)
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
@pytest.mark.parametrize("n", [1536, 6144])
@pytest.mark.parametrize("t", [0, 1, 7, 256])
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
torch.manual_seed(0)
dev = "cuda"
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
w = torch.randn(n, device=dev).to(torch.bfloat16)
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
ref_emb = table[ids.long()]
# Gather + embed_norm (base model / MTP prefill embed path).
out = embed_rmsnorm(ids, table, w, EPS)
assert out.shape == (t, n)
assert torch.equal(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
# Pure gather (use_embed_norm=False / replicated module forward).
out = embed_rmsnorm(ids, table, None, EPS)
assert torch.equal(out, ref_emb)
# Chained first-layer attn_norm (the target text-path forward): one launch
# emits both the residual and layer 0's normed attention input.
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
res, attn_in = embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain)
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
assert torch.equal(res, ref_res)
assert torch.equal(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
# Chained without embed_norm (use_embed_norm=False).
res, attn_in = embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain)
assert torch.equal(res, ref_emb)
assert torch.equal(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
-258
View File
@@ -1,258 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.models.inkling.nvidia.ops.sconv as sconv
from vllm.models.inkling.nvidia.ops import qkvr_prep
from vllm.platforms import current_platform
_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
def _make_inputs(*, is_local: bool, tokens: int = 33, tp_size: int = 4):
torch.manual_seed(0)
heads = 64 // tp_size
kv_heads = (16 if is_local else 8) // tp_size
head_dim = 128
d_rel = 16
rel_extent = 512 if is_local else 1024
page_size = 16
num_blocks = (tokens + page_size - 1) // page_size
q_width = heads * head_dim
kv_width = kv_heads * head_dim
r_width = heads * d_rel
device = "cuda"
qkvr = torch.randn(
tokens,
q_width + 2 * kv_width + r_width,
device=device,
dtype=torch.bfloat16,
)
k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16)
v_weight = torch.randn_like(k_weight)
q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16)
k_norm_weight = torch.randn_like(q_norm_weight)
rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16)
conv_cache = torch.zeros(
num_blocks,
kv_heads,
page_size,
2 * head_dim,
device=device,
dtype=torch.bfloat16,
)
key_cache = torch.empty(
num_blocks,
page_size,
kv_heads,
head_dim,
device=device,
dtype=torch.bfloat16,
)
value_cache = torch.empty_like(key_cache)
positions = torch.arange(tokens, device=device, dtype=torch.int64)
block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None]
seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32)
slots = torch.arange(tokens, device=device, dtype=torch.int64)
query_start = torch.zeros(tokens, device=device, dtype=torch.int32)
log_scaling_n_floor = None if is_local else 128000
log_scaling = None
if log_scaling_n_floor is not None:
log_scaling = torch.linspace(
1.0,
1.1,
tokens,
device=device,
dtype=torch.float32,
)
return (
qkvr,
k_weight,
v_weight,
q_norm_weight,
k_norm_weight,
rel_proj,
1e-6,
heads,
kv_heads,
head_dim,
d_rel,
conv_cache,
key_cache,
value_cache,
positions,
block_table,
seq_idx,
slots,
query_start,
slots,
0,
head_dim,
page_size,
log_scaling,
)
def _reference(args):
qkvr = args[0]
heads, kv_heads, head_dim, d_rel = args[7:11]
q_width = heads * head_dim
kv_width = kv_heads * head_dim
q, k, v, r = qkvr.split((q_width, kv_width, kv_width, heads * d_rel), dim=1)
conv_cache = args[11].clone()
k = sconv.fused_sconv(
k.contiguous(),
args[1],
conv_cache,
args[14],
args[15],
args[16],
args[17],
args[18],
args[20],
head_dim,
args[22],
)
v = sconv.fused_sconv(
v.contiguous(),
args[2],
conv_cache,
args[14],
args[15],
args[16],
args[17],
args[18],
args[21],
head_dim,
args[22],
)
def rms_norm(x, weight):
x = x.reshape(-1, head_dim).float()
rstd = torch.rsqrt(x.square().mean(1, keepdim=True) + args[6])
return (x * rstd * weight.float()).to(qkvr.dtype)
q = rms_norm(q, args[3]).view(qkvr.shape[0], heads, head_dim)
k = rms_norm(k, args[4]).view(qkvr.shape[0], kv_heads, head_dim)
v = v.view(qkvr.shape[0], kv_heads, head_dim)
rel = torch.mm(r.reshape(-1, d_rel), args[5]).view(qkvr.shape[0], heads, -1)
if args[23] is not None:
q = (q.float() * args[23][:, None, None]).to(q.dtype)
rel = (rel.float() * args[23][:, None, None]).to(rel.dtype)
key_cache = args[12].clone()
value_cache = args[13].clone()
slots = args[19]
valid = slots >= 0
key_cache.view(-1, kv_heads, head_dim)[slots[valid]] = k[valid]
value_cache.view(-1, kv_heads, head_dim)[slots[valid]] = v[valid]
return q.flatten(1), rel, key_cache, value_cache
@pytest.mark.skipif(
_cap is None,
reason="Inkling QKVR prep kernels require CUDA",
)
@pytest.mark.parametrize(
("is_local", "tokens", "tp_size"),
[
(True, 9, 4),
(False, 33, 4),
(True, 33, 8),
(True, 128, 4),
(False, 128, 8),
(True, 512, 4),
(False, 640, 8),
],
)
@torch.inference_mode()
def test_qkvr_prep_matches_reference(is_local, tokens, tp_size):
args = _make_inputs(is_local=is_local, tokens=tokens, tp_size=tp_size)
ref_q, ref_rel, ref_key, ref_value = _reference(args)
q, rel = qkvr_prep.fused_qkvr_prep(*args)
torch.testing.assert_close(q, ref_q, rtol=0.01, atol=0.02)
torch.testing.assert_close(rel, ref_rel, rtol=0.02, atol=0.125)
torch.testing.assert_close(args[12], ref_key, rtol=0.01, atol=0.01)
torch.testing.assert_close(args[13], ref_value, rtol=0, atol=0)
@pytest.mark.skipif(
_cap is None,
reason="Inkling QKVR prep kernels require CUDA",
)
@pytest.mark.parametrize("tokens", [9, 128])
@torch.inference_mode()
def test_qkvr_log_scaling_preserves_bf16_norm_output(tokens):
args = list(_make_inputs(is_local=False, tokens=tokens, tp_size=8))
tau = args[23]
assert tau is not None
scaled_q, scaled_rel = qkvr_prep.fused_qkvr_prep(*args)
args[23] = None
unscaled_q, unscaled_rel = qkvr_prep.fused_qkvr_prep(*args)
expected_q = (unscaled_q.float() * tau[:, None]).to(unscaled_q.dtype)
expected_rel = (unscaled_rel.float() * tau[:, None, None]).to(unscaled_rel.dtype)
torch.testing.assert_close(scaled_q, expected_q, rtol=0, atol=0)
torch.testing.assert_close(scaled_rel, expected_rel, rtol=0, atol=0)
@pytest.mark.skipif(
_cap is None,
reason="Inkling QKVR prep kernels require CUDA",
)
@pytest.mark.parametrize("tokens", [9, 128])
@torch.inference_mode()
def test_qkvr_prep_negative_slots_skip_cache_writes(tokens):
args = list(_make_inputs(is_local=True, tokens=tokens))
args[17].fill_(-1)
args[19].fill_(-1)
conv_cache = args[11].clone()
key_cache = args[12].clone()
value_cache = args[13].clone()
qkvr_prep.fused_qkvr_prep(*args)
torch.testing.assert_close(args[11], conv_cache, rtol=0, atol=0)
torch.testing.assert_close(args[12], key_cache, rtol=0, atol=0)
torch.testing.assert_close(args[13], value_cache, rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@torch.inference_mode()
def test_fused_sconv_negative_slots_skip_cache_writes():
args = list(_make_inputs(is_local=True, tokens=9))
args[17].fill_(-1)
cache = args[11].clone()
sconv.fused_sconv(
args[0][:, 16 * 128 : 16 * 128 + 4 * 128].contiguous(),
args[1],
args[11],
args[14],
args[15],
args[16],
args[17],
args[18],
0,
128,
args[22],
)
torch.testing.assert_close(args[11], cache, rtol=0, atol=0)
@pytest.mark.parametrize(
("rel_extent", "last_latency_rows", "first_throughput_rows"),
[(512, 8191, 8192), (1024, 2047, 2048)],
)
def test_rel_projection_schedule_crossover(
rel_extent, last_latency_rows, first_throughput_rows
):
assert not qkvr_prep.use_rel_proj_throughput(last_latency_rows, rel_extent)
assert qkvr_prep.use_rel_proj_throughput(first_throughput_rows, rel_extent)
-119
View File
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Parity test for the fused sconv seq-metadata kernel.
``sconv_seq_metadata`` fills the per-token ``seq_idx`` (owning request) and
``query_start`` (first x-row of that request) buffers in a single launch. Actual
tokens must match the searchsorted-based reference, while CUDA graph padding
must be initialized to safe zero values.
"""
import pytest
import torch
from vllm.models.inkling.nvidia.ops.sconv import sconv_seq_metadata
from vllm.models.inkling.nvidia.sconv_swa_attn import InklingSconvMetadataBuilder
CASES = [
# (query_lens, extra_pad_tokens)
([1], 0), # bsz1 decode
([1] * 8, 0), # uniform decode
([1] * 8, 3), # uniform decode, padded tokens past the last request
([2] * 4, 0), # uniform spec-decode
([2048], 0), # single prefill
([517, 1, 1, 33, 1, 128], 0), # mixed prefill/decode
([517, 1, 1, 33, 1, 128], 5), # mixed, padded
([1] * 500, 0), # many requests (deep binary search)
]
def _ref(query_start_loc: torch.Tensor, num_reqs: int, num_tokens: int):
cu_seqlens = query_start_loc[: num_reqs + 1].to(torch.int64)
token_idx = torch.arange(num_tokens, device=cu_seqlens.device, dtype=torch.int64)
seq_idx = (torch.searchsorted(cu_seqlens, token_idx, right=True) - 1).clamp(
max=num_reqs - 1
)
return seq_idx.to(torch.int32), cu_seqlens[seq_idx].to(torch.int32)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("query_lens,extra_pad", CASES)
def test_sconv_seq_metadata_matches_searchsorted(query_lens, extra_pad):
device = "cuda"
num_reqs = len(query_lens)
query_start_loc = torch.tensor(
[0] + list(torch.tensor(query_lens).cumsum(0)), dtype=torch.int32
).to(device)
num_actual_tokens = int(query_start_loc[-1])
num_padded_tokens = num_actual_tokens + extra_pad
ref_seq, ref_qs = _ref(query_start_loc, num_reqs, num_actual_tokens)
seq_idx = torch.full((num_padded_tokens,), -1, dtype=torch.int32, device=device)
query_start = torch.full_like(seq_idx, -1)
sconv_seq_metadata(
query_start_loc,
num_reqs,
num_actual_tokens,
seq_idx,
query_start,
num_padded_tokens,
)
torch.testing.assert_close(seq_idx[:num_actual_tokens], ref_seq, rtol=0, atol=0)
torch.testing.assert_close(query_start[:num_actual_tokens], ref_qs, rtol=0, atol=0)
assert torch.count_nonzero(seq_idx[num_actual_tokens:]) == 0
assert torch.count_nonzero(query_start[num_actual_tokens:]) == 0
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sconv_metadata_reuses_padded_static_buffers():
device = torch.device("cuda")
builder = object.__new__(InklingSconvMetadataBuilder)
builder.seq_idx_buffer = torch.empty(8, dtype=torch.int32, device=device)
builder.query_start_buffer = torch.empty(8, dtype=torch.int32, device=device)
class CommonMetadata:
num_reqs = 1
num_actual_tokens = 8
query_start_loc = torch.tensor([0, 3], dtype=torch.int32, device=device)
query_start_loc_cpu = torch.tensor([0, 3], dtype=torch.int32)
block_table_tensor = torch.zeros((1, 1), dtype=torch.int32, device=device)
slot_mapping = torch.tensor(
[0, 1, 2, -1, -1, -1, -1, -1],
dtype=torch.int64,
device=device,
)
common = CommonMetadata()
first = builder.build(0, common)
pointers = tuple(
tensor.data_ptr()
for tensor in (
first.block_table,
first.slot_mapping,
first.seq_idx,
first.query_start,
)
)
common.query_start_loc[1] = 5
common.query_start_loc_cpu[1] = 5
common.slot_mapping[:5] = torch.arange(5, dtype=torch.int64, device=device)
second = builder.build(0, common)
assert second.slot_mapping.shape == (8,)
assert second.seq_idx.shape == (8,)
assert second.query_start.shape == (8,)
assert pointers == tuple(
tensor.data_ptr()
for tensor in (
second.block_table,
second.slot_mapping,
second.seq_idx,
second.query_start,
)
)
assert torch.count_nonzero(second.seq_idx[5:]) == 0
assert torch.count_nonzero(second.query_start[5:]) == 0
assert torch.all(second.slot_mapping[5:] == -1)
-2
View File
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-296
View File
@@ -1,296 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AMD AttnRes requires ROCm",
)
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
storage = torch.randn(
*shape[:-1],
shape[-1] + padding,
device="cuda",
dtype=torch.bfloat16,
)
return storage[..., : shape[-1]]
def _reference(
prefix: torch.Tensor,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
num_blocks: int,
eps: float,
) -> torch.Tensor:
hidden_size = prefix.shape[-1]
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
keys = F.rms_norm(values, (hidden_size,), norm_weight, eps)
probs = (keys @ qk_weight).softmax(dim=-1)
return torch.matmul(probs.unsqueeze(1), values).squeeze(1)
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"block_capacity",
"hidden_size",
"row_padding",
),
[
pytest.param(0, 3, 5, 128, 0, id="empty"),
pytest.param(1, 1, 2, 128, 0, id="decode-single"),
pytest.param(17, 4, 6, 1024, 7, id="decode-padded"),
pytest.param(320, 8, 10, 7168, 0, id="prefill-full"),
],
)
def test_amd_attn_res_matches_reference(
num_tokens: int,
num_blocks: int,
block_capacity: int,
hidden_size: int,
row_padding: int,
) -> None:
eps = 1e-5
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=row_padding)
blocks = _randn_with_row_padding(
num_tokens,
block_capacity,
hidden_size,
padding=row_padding,
)
norm_weight = 1 + 0.1 * torch.randn(
hidden_size, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
)
expected = _reference(
prefix,
blocks,
norm_weight,
qk_weight,
num_blocks,
eps,
)
original_prefix = prefix.clone()
original_blocks = blocks.clone()
actual = attn_res(
prefix,
None,
blocks,
norm_weight,
qk_weight,
None,
num_blocks,
-1,
eps,
0.0,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, original_prefix, atol=0, rtol=0)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.shape == prefix.shape
assert actual.is_contiguous()
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"hidden_size",
"has_delta",
"write_block",
"apply_output_norm",
),
[
pytest.param(1, 0, 128, False, True, True, id="empty-write-norm"),
pytest.param(7, 1, 1024, True, False, True, id="single-add-norm"),
pytest.param(17, 5, 7168, True, True, True, id="padded-write-add"),
pytest.param(3, 8, 7168, True, False, True, id="full-add-norm"),
pytest.param(320, 4, 7168, True, False, False, id="prefill-add"),
],
)
def test_amd_attn_res_fused_contract(
num_tokens: int,
num_blocks: int,
hidden_size: int,
has_delta: bool,
write_block: bool,
apply_output_norm: bool,
) -> None:
torch.manual_seed(42)
eps = 1e-5
output_eps = 2e-5
block_capacity = 9
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
delta = (
_randn_with_row_padding(num_tokens, hidden_size, padding=11)
if has_delta
else None
)
blocks = _randn_with_row_padding(
num_tokens, block_capacity, hidden_size, padding=13
)
norm_weight = 1 + 0.1 * torch.randn(
hidden_size, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
)
output_norm_weight = (
1 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16)
if apply_output_norm
else None
)
expected_prefix = prefix.clone()
if delta is not None:
expected_prefix = expected_prefix + delta
values = torch.cat(
(blocks[:, :num_blocks].clone(), expected_prefix.unsqueeze(1)), dim=1
)
keys = F.rms_norm(values.float(), (hidden_size,), norm_weight.float(), eps)
probs = (keys @ qk_weight.float()).softmax(dim=-1)
expected = torch.matmul(probs.unsqueeze(1), values.float()).squeeze(1)
if output_norm_weight is not None:
expected = F.rms_norm(
expected, (hidden_size,), output_norm_weight.float(), output_eps
)
expected = expected.to(prefix.dtype)
original_blocks = blocks.clone()
block_write_idx = num_blocks if write_block else -1
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
block_write_idx,
eps,
output_eps,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
if write_block:
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize(
"num_tokens,num_blocks,has_delta,write_block",
[
(0, 0, False, False),
(1, 0, True, True),
(17, 4, True, True),
(320, 8, False, False),
],
)
def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
num_tokens,
num_blocks,
has_delta,
write_block,
default_vllm_config,
):
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
torch.manual_seed(41)
hidden_size = 7168
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None
blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13)
norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1
score = torch.randn_like(norm) / hidden_size**0.5
out_norm = 1 + torch.randn_like(norm) * 0.1
ref_prefix, ref_blocks = prefix.clone(), blocks.clone()
kwargs = dict(
num_blocks=num_blocks,
block_write_idx=num_blocks if write_block else -1,
eps=1e-5,
output_norm_eps=1e-5,
)
reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs)
output, scale = attn_res(
prefix,
delta,
blocks,
norm,
score,
out_norm,
quant_dtype=current_platform.fp8_dtype(),
**kwargs,
)
assert output.dtype == current_platform.fp8_dtype()
assert scale.shape == (num_tokens, 1)
if num_tokens:
# Keep reference rounding independent of Inductor's fused quantization.
quant = QuantFP8(
static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False
)
ref_output, ref_scale = quant(reference)
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
# Floating-point fusion can move values at FP8 rounding midpoints.
# Bound each change to the adjacent code and bound how often it occurs.
# Codes are sign-magnitude, so order them by value to keep a step
# across zero adjacent.
def ordinal(x: torch.Tensor) -> torch.Tensor:
code = x.view(torch.uint8).to(torch.int16)
return torch.where(code >= 0x80, 0x80 - code, code)
code_delta = (ordinal(output) - ordinal(ref_output)).abs()
assert code_delta.max().item() <= 1
assert (code_delta != 0).float().mean().item() < 1e-4
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0)
@pytest.mark.parametrize(
"matching,requires_unquantized", [(True, False), (False, False), (True, True)]
)
def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized):
from types import SimpleNamespace
from torch import nn
from vllm.model_executor.layers.fusion.quant_activation import (
expose_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper)
nn.Module.__init__(attention)
attention.indexer = None
attention.q_lora_rank = 1536
attention.fused_qkv_a_proj = nn.Identity()
attention.g_proj = nn.Identity()
kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym)
expose_input_quant_key(attention.fused_qkv_a_proj, kernel)
if matching:
expose_input_quant_key(attention.g_proj, kernel)
attention.g_proj.requires_unquantized_input = requires_unquantized
actual = attention.get_input_quant_key()
assert actual == (
kFp8DynamicTokenSym if matching and not requires_unquantized else None
)
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
Adaptive verification splits the draft budget evenly on the host, then trims
each verify request on device. The KDA spec path must therefore plan from the
device offsets, never from host lengths that merely look uniform.
"""
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import SpeculativeConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.models.kimi_k3.amd.kda_metadata import (
KimiK3ROCmKDABackend,
KimiK3ROCmKDAMetadataBuilder,
)
from vllm.v1.kv_cache_interface import MambaSpec
BLOCK_SIZE = 16
DEVICE = torch.device("cpu")
NUM_SPEC = 3
def _config(adaptive: bool):
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=NUM_SPEC,
enable_adaptive_verification=adaptive,
)
vllm_config.cache_config.mamba_cache_mode = "none"
return vllm_config
def _spec() -> MambaSpec:
return MambaSpec(
block_size=BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
num_speculative_blocks=NUM_SPEC,
)
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
return KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=_spec(),
layer_names=["layer.0"],
vllm_config=_config(adaptive),
device=DEVICE,
)
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
)
common = common.replace(
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
)
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
def test_adaptive_verification_plans_from_device_offsets():
"""Equal host lengths would select the fixed-length recurrent kernel,
which places sequence i at i * L and ignores the device cu_seqlens."""
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
assert bound(_config(adaptive=False), _spec()) is None
meta = _trimmed_verify_batch(_builder(adaptive=True))
assert meta.num_spec_decodes == 2
assert meta.uniform_spec_sequence_length is None
assert torch.equal(
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
)
# Without adaptive verification the host lengths are exact and kept.
meta = _trimmed_verify_batch(_builder(adaptive=False))
assert meta.uniform_spec_sequence_length == 3
@@ -1,527 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm KDA prefill checkpoint: where it is taken and what it stores.
Two things have to line up for a checkpoint to be safe: the metadata must name
a position the chunk kernel will actually write, and the conv window saved
there must be the one a resumed prefill would have left.
"""
import pytest
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDAMetadataBuilder
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
)
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
KDA_CHECKPOINT_ALIGNMENT,
is_fused_kda_chunk_supported,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
pytestmark = pytest.mark.skipif(
not (
current_platform.is_rocm()
and torch.cuda.is_available()
and is_fused_kda_chunk_supported()
),
reason="The ROCm KDA prefill checkpoint is exported by the fused chunk "
"kernel, which needs a gfx950 build that includes it",
)
# A mamba block size the chunk walk lands on. The real K3 value is 768; any
# multiple of the kernel's 64-token chunk behaves the same and keeps the
# fixtures small.
BLOCK_SIZE = 64
class _StubSpeculativeConfig:
method = "dspark"
parallel_drafting = False
def __init__(self, num_speculative_tokens: int) -> None:
self.num_speculative_tokens = num_speculative_tokens
def use_eagle_block_drop(self) -> bool:
return False
def _builder(
num_prefill_checkpoint_blocks: int = 1,
mamba_cache_mode: str = "align",
block_size: int = BLOCK_SIZE,
prefix_match_unit: int | None = None,
num_spec: int = 0,
) -> KimiK3ROCmKDAMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=block_size
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
if num_spec:
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
builder = KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=block_size,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
KDA_CHECKPOINT_ALIGNMENT if num_prefill_checkpoint_blocks else None
),
num_speculative_blocks=num_spec,
),
layer_names=["layer.0"],
vllm_config=vllm_config,
device=torch.device("cuda"),
)
vllm_config.cache_config.hash_block_size = prefix_match_unit or block_size
vllm_config.cache_config.cache_hit_alignment_tokens = (
prefix_match_unit or block_size
)
return builder
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
common = create_common_attn_metadata(
batch, block_size, torch.device("cuda"), arange_block_indices=True
)
return _builder(block_size=block_size, **kw).build(0, common)
def _i32(values: list[int]) -> torch.Tensor:
return torch.tensor(values, dtype=torch.int32, device="cuda")
def test_checkpoint_metadata_targets_last_aligned_boundary() -> None:
# Slot p holds the state after (p + 1) * block_size tokens, so a 200-token
# prompt checkpoints at 192 into slot 2 and a 128-token one at 64 into slot
# 0. The block table is an arange, so request 1 owns blocks 4..7.
md = _build(BatchSpec(seq_lens=[200, 128], query_lens=[200, 128]))
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 64]))
torch.testing.assert_close(md.checkpoint.state_indices, _i32([2, 4]))
def test_checkpoint_offset_is_relative_to_query_start() -> None:
# 256 tokens computed, 144 more this step: the boundary at 384 lands 128
# tokens into the chunk, and cdiv(400, 64) - 2 = 5 is its slot.
md = _build(BatchSpec(seq_lens=[400, 64], query_lens=[144, 64]))
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([128, 0]))
torch.testing.assert_close(md.checkpoint.state_indices, _i32([5, NULL_BLOCK_ID]))
def test_checkpoint_metadata_avoids_live_state_row() -> None:
# Writing the row the walk itself owns would race it.
md = _build(BatchSpec(seq_lens=[200, 130], query_lens=[200, 130]))
assert md.checkpoint is not None
# `mamba_get_block_table_tensor` hands the walk block (seq_len - 1) // bs.
live = torch.tensor([(200 - 1) // BLOCK_SIZE, (130 - 1) // BLOCK_SIZE])
table = torch.arange(2 * 4).view(2, 4)
live_rows = table[torch.arange(2), live].cuda()
assert not bool((md.checkpoint.state_indices == live_rows).any())
def test_checkpoint_metadata_skips_chunk_without_boundary() -> None:
# 128 computed, 40 more: the only boundary at 128 is the state this chunk
# started from, so there is nothing new to snapshot.
assert _build(BatchSpec(seq_lens=[168], query_lens=[40])).checkpoint is None
def test_checkpoint_metadata_skips_null_reservation() -> None:
# Only the chunk that finishes a prompt gets a block reserved; every
# earlier one leaves the shared null block, which nothing hashes and
# writing it would corrupt every other request's scratch row.
batch = BatchSpec(seq_lens=[200], query_lens=[200])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
)
# cdiv(200, 64) - 2 = 2 is the checkpoint slot.
common.block_table_tensor[0, 2] = NULL_BLOCK_ID
md = _builder().build(0, common)
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.state_indices, _i32([NULL_BLOCK_ID]))
def test_checkpoint_metadata_absent_without_reserved_block() -> None:
md = _build(
BatchSpec(seq_lens=[200], query_lens=[200]),
num_prefill_checkpoint_blocks=0,
)
assert getattr(md, "checkpoint", None) is None
def test_spec_checkpoint_metadata_gathers_non_spec_rows_by_index() -> None:
# gdn_attn selects the non-spec rows by mask and reclassifies their
# one-token decodes as prefills, so the chunk kernel is handed rows 1, 3
# and 4 here while a num_decodes: slice would hand it rows 0, 1 and 2.
num_spec = 2
batch = BatchSpec(
seq_lens=[300, 200, 400, 512, 128],
query_lens=[num_spec + 1, 200, num_spec + 1, 1, 128],
)
common = create_common_attn_metadata(
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
)
# Align mode reads 1 + num_speculative_blocks columns, so widen the table.
rows, cols = common.block_table_tensor.shape
width = cols + num_spec
common.block_table_tensor = torch.arange(
rows * width, dtype=torch.int32, device="cuda"
).view(rows, width)
md = _builder(num_spec=num_spec).build(
0,
common,
num_accepted_tokens=torch.ones(rows, dtype=torch.int32, device="cuda"),
num_decode_draft_tokens_cpu=torch.tensor(
[num_spec, -1, num_spec, -1, -1], dtype=torch.int32
),
)
assert md.spec_sequence_masks is not None
assert md.num_decodes == 0 and md.num_prefills == 3
assert md.checkpoint is not None
# Row 3 resumes mid-block, so its chunk holds no reusable boundary.
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 0, 64]))
# Request r owns blocks r*width.., so slot cdiv(200, 64) - 2 = 2 for
# request 1 and cdiv(128, 64) - 2 = 0 for request 4.
torch.testing.assert_close(
md.checkpoint.state_indices, _i32([width + 2, NULL_BLOCK_ID, 4 * width])
)
def test_checkpoint_metadata_absent_outside_align_mode() -> None:
md = _build(
BatchSpec(seq_lens=[200], query_lens=[200]),
mamba_cache_mode="none",
)
assert getattr(md, "checkpoint", None) is None
# ---------------------------------------------------------------------------
# Conv half
# ---------------------------------------------------------------------------
DIM = 384
WIDTH = 4
STATE_LEN = WIDTH - 1
def store_conv_checkpoints(x, conv_state, cu, offsets, rows, state_len) -> None:
KimiK3ROCmKDAPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(offsets, rows),
raw_qkv=x,
conv_state=conv_state,
cu_seqlens=cu,
)
def _conv_fixture(
seqlens: list[int], slots: int = 16, seed: int = 0, num_spec: int = 0
):
# kda_state_shape widens the conv row to conv_kernel_size - 1 + num_spec,
# but the window every reader starts from stays the leading STATE_LEN.
torch.manual_seed(seed)
total = sum(seqlens)
x = torch.randn(total, DIM, device="cuda", dtype=torch.bfloat16)
cu = torch.tensor(
[0] + torch.tensor(seqlens).cumsum(0).tolist(),
device="cuda",
dtype=torch.int32,
)
conv_state = torch.full(
(slots, DIM, STATE_LEN + num_spec),
float("nan"),
device="cuda",
dtype=torch.bfloat16,
)
return x, cu, conv_state
def test_conv_checkpoint_saves_window_before_offset() -> None:
seqlens = [200, 130]
x, cu, conv_state = _conv_fixture(seqlens)
offsets = _i32([192, 128])
rows = _i32([5, 9])
store_conv_checkpoints(x, conv_state, cu, offsets, rows, STATE_LEN)
for seq, (offset, row) in enumerate(zip([192, 128], [5, 9])):
end = int(cu[seq]) + offset
expected = x[end - STATE_LEN : end].transpose(0, 1)
torch.testing.assert_close(conv_state[row], expected)
def test_conv_checkpoint_skips_opted_out_sequences() -> None:
x, cu, conv_state = _conv_fixture([200, 200])
before = conv_state.clone()
store_conv_checkpoints(
x, conv_state, cu, _i32([0, 192]), _i32([5, NULL_BLOCK_ID]), STATE_LEN
)
assert torch.equal(conv_state.isnan(), before.isnan())
def test_exporter_stores_conv_window_through_shared_interface() -> None:
x, cu, conv_state = _conv_fixture([200, 200])
exporter = KimiK3ROCmKDAPrefillCheckpointExporter(state_len=STATE_LEN)
assert isinstance(exporter, MambaPrefillCheckpointExporter)
checkpoint = MambaPrefillCheckpointMetadata(
_i32([192, 0]), _i32([5, NULL_BLOCK_ID])
)
exporter.export(checkpoint, raw_qkv=x, conv_state=conv_state, cu_seqlens=cu)
torch.testing.assert_close(conv_state[5], x[192 - STATE_LEN : 192].transpose(0, 1))
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
untouched[5] = False
assert bool(conv_state[untouched].isnan().all())
@pytest.mark.parametrize("num_spec", [0, 2, 8])
def test_conv_checkpoint_matches_prefill_truncated_at_offset(num_spec: int) -> None:
# causal_conv1d_fn's own state write is the reference: a prefill that
# stopped at the offset leaves exactly the row a later prefix-cache hit
# reads back. Speculative decoding widens the row without moving that
# window, so a checkpoint sized off the row would sit num_spec tokens in
# the past.
seqlens = [200, 130]
offsets = [192, 64]
x, cu, conv_state = _conv_fixture(seqlens, seed=3, num_spec=num_spec)
weight = torch.randn(DIM, WIDTH, device="cuda", dtype=torch.bfloat16) * 0.3
bias = torch.randn(DIM, device="cuda", dtype=torch.bfloat16)
rows = [5, 9]
store_conv_checkpoints(x, conv_state, cu, _i32(offsets), _i32(rows), STATE_LEN)
# A prefill of just the prefix, into its own cache. Row 0 is the null block
# the conv kernel skips by design, so the reference rows start at 1.
prefix = torch.cat(
[x[int(cu[n]) : int(cu[n]) + off] for n, off in enumerate(offsets)]
)
prefix_cu = torch.tensor(
[0] + torch.tensor(offsets).cumsum(0).tolist(),
device="cuda",
dtype=torch.int32,
)
reference = torch.zeros_like(conv_state)
causal_conv1d_fn(
prefix.transpose(0, 1),
weight,
bias,
activation="silu",
conv_states=reference,
has_initial_state=torch.zeros(2, dtype=torch.bool, device="cuda"),
cache_indices=_i32([1, 2]),
query_start_loc=prefix_cu,
)
# causal_conv1d_fn writes only the leading STATE_LEN columns.
window = slice(None), slice(None, STATE_LEN)
torch.testing.assert_close(conv_state[rows[0]][window], reference[1][window])
torch.testing.assert_close(conv_state[rows[1]][window], reference[2][window])
def test_conv_checkpoint_leaves_spec_tail_of_row_alone() -> None:
# The draft-token columns belong to causal_conv1d_update. Filling the whole
# row would hand every reader, which indexes from column 0, a window
# shifted num_spec tokens into the past.
num_spec = 4
x, cu, conv_state = _conv_fixture([200], num_spec=num_spec, seed=11)
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([5]), STATE_LEN)
torch.testing.assert_close(
conv_state[5, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
)
assert bool(conv_state[5, :, STATE_LEN:].isnan().all()), (
"the checkpoint wrote into the speculative-decode tail of the row"
)
def test_conv_checkpoint_honours_transposed_cache_view() -> None:
# The SD conv layout reaches the layer transposed, not contiguous.
x, cu, _ = _conv_fixture([200])
raw = torch.full(
(16, STATE_LEN, DIM), float("nan"), device="cuda", dtype=torch.bfloat16
)
conv_state = raw.transpose(-1, -2)
assert not conv_state.is_contiguous()
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([7]), STATE_LEN)
torch.testing.assert_close(conv_state[7], x[192 - STATE_LEN : 192].transpose(0, 1))
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
untouched[7] = False
assert bool(raw[untouched].isnan().all())
def test_checkpoint_metadata_indexes_prefill_rows_after_decodes() -> None:
# The chunk kernel receives the prefill tail alone, so the metadata has to
# be indexed the same way or it snapshots the wrong request's boundary.
md = _build(BatchSpec(seq_lens=[64, 200], query_lens=[1, 200]))
assert md.num_decodes == 1 and md.num_prefills == 1
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192]))
# Request 1 owns blocks 4..7, so its slot-2 state block is 6.
torch.testing.assert_close(md.checkpoint.state_indices, _i32([6]))
def test_conv_checkpoint_addresses_row_past_int32_limit() -> None:
# state_stride_0 is the whole mamba page, padded up to the attention page,
# so row * stride passes 2**31 a few thousand blocks into a large cache and
# an int32 wrap stores outside the tensor. The margin on the plain prefill
# path is 1.6x, which is luck: a bigger cache, a higher utilization or the
# wider row speculative decoding allocates all close it.
stride_row = DIM * (STATE_LEN + 8)
row = 2**31 // stride_row + 1
need = (row + 1) * stride_row * 2
free, _ = torch.accelerator.get_memory_info()
if free < need + (4 << 30):
pytest.skip(f"needs {need >> 30} GiB of free VRAM to reach the wrap")
x, cu, _ = _conv_fixture([200])
conv_state = torch.zeros(
row + 1, DIM, STATE_LEN + 8, device="cuda", dtype=torch.bfloat16
)
assert conv_state.stride(0) * row > 2**31, "fixture does not reach the wrap"
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([row]), STATE_LEN)
torch.testing.assert_close(
conv_state[row, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
)
assert not bool(conv_state[:row].any()), "the store wrote outside its row"
def test_conv_checkpoint_reads_row_strided_projection() -> None:
# mixed_qkv is a band of the fused QKVGFAB projection, so its row stride is
# the whole projection width, not DIM.
torch.manual_seed(5)
total, pad = 200, 177
projection = torch.randn(total, DIM + pad, device="cuda", dtype=torch.bfloat16)
x = projection[:, :DIM]
assert x.stride(0) == DIM + pad
cu = _i32([0, total])
conv_state = torch.full(
(16, DIM, STATE_LEN), float("nan"), device="cuda", dtype=torch.bfloat16
)
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([4]), STATE_LEN)
torch.testing.assert_close(conv_state[4], x[192 - STATE_LEN : 192].transpose(0, 1))
def test_finer_prefix_match_unit_moves_checkpoint_to_hash_grid() -> None:
# A partial prefix hit matches at prefix_match_unit, so a state captured
# only at block boundaries is too coarse to resume from. Block 256 with
# unit 128 moves a 700-token prompt's checkpoint from 512 to 640.
md = _build(
BatchSpec(seq_lens=[700], query_lens=[700]),
block_size=256,
prefix_match_unit=128,
)
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([640]))
# Slot is still block-indexed: cdiv(700, 256) - 2 = 1, request 0 owns 0..2.
torch.testing.assert_close(md.checkpoint.state_indices, _i32([1]))
def test_checkpoint_metadata_skips_unaligned_offset() -> None:
# The scheduler declines a boundary off the published alignment and
# reserves nothing, so the builder has to reach the same verdict or it
# writes a block nobody hashed.
# Block 192, unit 96: a 700-token prompt's last hash boundary is 672, which
# is 32 tokens off the chunk grid.
md = _build(
BatchSpec(seq_lens=[700], query_lens=[700]),
block_size=192,
prefix_match_unit=96,
)
assert md.checkpoint is None
# ---------------------------------------------------------------------------
# Opting in
# ---------------------------------------------------------------------------
def _spec_for(monkeypatch, **attrs) -> MambaSpec:
"""Run the layer's `get_kv_cache_spec` over a stubbed base spec."""
base = MambaSpec(block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,))
monkeypatch.setattr(
GatedDeltaNetAttention, "get_kv_cache_spec", lambda self, cfg: base
)
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
for name, value in dict(
use_prefill_checkpoint=True,
kda_prefill_backend="fused",
num_spec=0,
use_safe_gate=True,
).items():
setattr(layer, name, attrs.get(name, value))
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
vllm_config.cache_config.mamba_cache_mode = "align"
return layer.get_kv_cache_spec(vllm_config)
def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
# The scheduler learns the kernel's granularity only from the spec, so
# reserving a block without publishing the alignment would hash a state
# that is never written.
spec = _spec_for(monkeypatch)
assert spec.num_prefill_checkpoint_blocks == 1
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
@pytest.mark.parametrize(
"attr,value",
[
("use_prefill_checkpoint", False),
("kda_prefill_backend", "triton"),
("kda_prefill_backend", "gluon"),
("use_safe_gate", False),
],
)
def test_paths_that_cannot_export_stay_opted_out(monkeypatch, attr, value) -> None:
spec = _spec_for(monkeypatch, **{attr: value})
assert spec.num_prefill_checkpoint_blocks == 0
assert spec.prefill_checkpoint_alignment is None
def test_speculative_decoding_does_not_opt_out(monkeypatch) -> None:
# num_spec reaches the kernels as a wider conv row and an interleaved
# batch, both of which the layer handles.
spec = _spec_for(monkeypatch, num_spec=8)
assert spec.num_prefill_checkpoint_blocks == 1
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
-930
View File
@@ -1,930 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The fused ROCm KDA chunk kernel must match the Triton chunk path.
The fused kernel reassociates nothing, but it keeps the chunk states in
registers and rounds them to bfloat16 only where the Triton path does, so the
outputs agree to bfloat16 rounding and the final states nearly exactly.
"""
import pytest
import torch
from vllm.platforms import current_platform
def _fused_chunk_available() -> bool:
if not (current_platform.is_rocm() and torch.cuda.is_available()):
return False
from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported
return is_fused_kda_chunk_supported()
pytestmark = pytest.mark.skipif(
not _fused_chunk_available(),
reason="The fused KDA chunk kernel needs a gfx950 build that includes it",
)
HEAD_DIM = 128
NUM_HEADS = 12
LOWER_BOUND = -5.0
def _requires_kernel() -> None:
if not hasattr(torch.ops._C, "fused_kda_chunk"):
pytest.skip("vLLM was built without the fused KDA chunk kernel")
def _inputs(seqlens: list[int], seed: int) -> dict:
torch.manual_seed(seed)
dev, dtype = "cuda", torch.bfloat16
total = sum(seqlens)
cu = torch.tensor(
[0] + torch.tensor(seqlens).cumsum(0).tolist(), device=dev, dtype=torch.int32
)
shape = (1, total, NUM_HEADS, HEAD_DIM)
return dict(
q=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
k=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
raw_g=torch.randn(shape, device=dev, dtype=dtype),
raw_beta=torch.randn(1, total, NUM_HEADS, device=dev, dtype=dtype),
A_log=torch.randn(NUM_HEADS, device=dev, dtype=torch.float32) * 0.5,
dt_bias=torch.randn(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32)
* 0.5,
h0=torch.randn(
len(seqlens), NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
)
* 0.1,
cu=cu,
)
def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
return chunk_kda_prefill(
q=inp["q"].clone(),
k=inp["k"].clone(),
v=inp["v"].clone(),
raw_g=inp["raw_g"].clone(),
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
g_bias=inp["dt_bias"],
lower_bound=LOWER_BOUND,
initial_state=inp["h0"].clone(),
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
backend=kw.pop("backend", "fused" if use_fused else "triton"),
out=out,
**kw,
)
def _force_groups(monkeypatch, groups: int) -> None:
"""Pin the group count, which is otherwise a function of the device."""
import vllm.models.kimi_k3.amd.ops.kda_chunk as kda_chunk
monkeypatch.setattr(kda_chunk, "_chunk_groups", lambda *_: groups)
@pytest.mark.parametrize(
"seqlens",
[
[1], # shorter than a chunk
[64], # exactly one chunk
[1024],
[4096], # many chunks, one sequence
[513, 64, 1, 1200], # ragged, including a partial tail chunk
[512] * 6,
],
)
def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
_requires_kernel()
inp = _inputs(seqlens, seed=len(seqlens))
o_ref, state_ref = _run(inp, use_fused=False)
o_fused, state_fused = _run(inp, use_fused=True)
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize(
"seqlens",
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
)
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
pytest.skip("AITER Gluon chunk_kda is not available here")
inp = _inputs(seqlens, seed=len(seqlens))
o_ref, state_ref = _run(inp, use_fused=False)
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
@pytest.mark.parametrize("use_fused", [False, True])
def test_out_argument_receives_the_result(use_fused: bool) -> None:
"""``out`` is a promise about where the result lands.
The layer passes a slice of its own output buffer so neither path has to
copy afterwards, and it then skips the copy based on the returned tensor
aliasing that buffer. Both must hold for the fused path and for the Triton
fallback, which allocates its own buffer internally.
"""
if use_fused:
_requires_kernel()
seqlens = [321, 64, 700]
inp = _inputs(seqlens, seed=13)
o_ref, _ = _run(inp, use_fused=use_fused)
# A non-zero offset into a larger buffer, as the layer produces when a
# mixed batch puts decode tokens first.
total = sum(seqlens)
buf = torch.empty(
1, total + 8, NUM_HEADS, HEAD_DIM, device="cuda", dtype=o_ref.dtype
)
view = buf[:, 8:]
o_got, _ = _run(inp, use_fused=use_fused, out=view)
assert o_got.data_ptr() == view.data_ptr()
torch.testing.assert_close(view.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
def test_fused_chunk_accepts_projection_views() -> None:
"""`beta` and `g` arrive as last-dim slices of the fused QKVGFAB projection.
Those views carry the projection's row stride, so a kernel that indexes them
densely reads the wrong elements or rejects the tensor outright. Build the
inputs the way the layer does rather than as standalone tensors.
"""
_requires_kernel()
seqlens = [777, 64, 391]
inp = _inputs(seqlens, seed=7)
total = sum(seqlens)
# One wide projection, split on the last dim exactly like KimiK3DeltaAttention.
qkv_width = 3 * NUM_HEADS * HEAD_DIM
projection = torch.randn(
total, qkv_width + NUM_HEADS, device="cuda", dtype=torch.bfloat16
)
beta_view = projection.split([qkv_width, NUM_HEADS], dim=-1)[1].unsqueeze(0)
assert not beta_view.is_contiguous()
inp["raw_beta"] = beta_view.contiguous()
o_ref, state_ref = _run(inp, use_fused=False)
inp["raw_beta"] = beta_view
o_fused, state_fused = _run(inp, use_fused=True)
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
def test_fused_chunk_survives_correlated_keys_and_real_gate_bias() -> None:
"""The input regime the served model actually produces.
This is what forces the intra-chunk inverse to keep fp32 MFMA operands: at
bf16 the Neumann doubling's intermediate powers cancel catastrophically
here, diverging by orders of magnitude and driving the final state to inf.
Three properties of real traffic combine here and in none of the tests
above, all measured from tensors captured off a live TP8 server:
* repetitive text gives near-duplicate L2-normalized keys, so ``k_i . k_j``
approaches 1 and the Neumann terms stop shrinking;
* the checkpoint's ``dt_bias`` lies in ``[-7.8, -1.4]``, which makes the
within-chunk gate cumsum span ~300 powers of two;
* ``beta`` saturates near 0.95.
``_inputs`` draws all three from ``randn``, landing in the benign regime.
"""
_requires_kernel()
torch.manual_seed(11)
dev, dtype = "cuda", torch.bfloat16
total = 1088
shape = (1, total, NUM_HEADS, HEAD_DIM)
base = torch.randn(1, 1, NUM_HEADS, HEAD_DIM, device=dev, dtype=dtype)
def correlated():
return (base + torch.randn(shape, device=dev, dtype=dtype) * 0.02).contiguous()
inp = dict(
q=correlated(),
k=correlated(),
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
raw_g=torch.randn(shape, device=dev, dtype=dtype),
raw_beta=torch.full((1, total, NUM_HEADS), 3.0, device=dev, dtype=dtype),
A_log=torch.rand(NUM_HEADS, device=dev, dtype=torch.float32) * 0.6 - 0.42,
dt_bias=torch.rand(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32) * 6.4
- 7.8,
h0=torch.zeros(
1, NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
),
cu=torch.tensor([0, total], device=dev, dtype=torch.int32),
)
o_ref, state_ref = _run(inp, use_fused=False)
o_fused, state_fused = _run(inp, use_fused=True)
# Looser than the tests above: near-identical keys plus a saturated beta
# are past anything real text produces, and the inverse is rounded to bf16
# before the w/u GEMMs. The tolerance still catches divergence, which in
# this regime is orders of magnitude rather than a few times the bound.
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=5e-2, atol=5e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=5e-2, atol=5e-2)
@pytest.mark.parametrize(
"seqlens",
[
[1], # every conv tap comes from the cache
[2], # cache and batch both contribute
[64],
[1024],
[513, 64, 1, 1200], # ragged, tail chunk shorter than the conv width
],
)
@pytest.mark.parametrize("has_state", [False, True])
def test_fused_conv_matches_reference_conv(seqlens: list[int], has_state: bool) -> None:
"""The prologue's optional fused convolution must match a separate conv.
The layer does not use this path -- recomputing the conv once per load
phase costs more than the ``causal_conv1d_fn`` launches it saves -- so it
is driven through the ops directly rather than through
``chunk_kda_prefill``, which does not expose it.
The reference is computed in torch rather than with ``causal_conv1d_fn``,
which is not self-consistent for these shapes on ROCm (it leaves part of
its output unwritten, so repeated identical calls disagree).
The cases that matter are the ones touching the sequence boundary, where
the fused path takes taps from the conv cache instead of the batch:
sequences shorter than the kernel width, and a cache that is or is not
populated. The rolled-forward cache is checked too, since a wrong shift
only shows up on the next step.
"""
import torch.nn.functional as F
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
fused_kda_chunk,
fused_kda_prologue,
)
_requires_kernel()
dev, dtype = "cuda", torch.bfloat16
lp = NUM_HEADS * HEAD_DIM
width, n = 4, len(seqlens)
inp = _inputs(seqlens, seed=7 + len(seqlens) + int(has_state))
total = int(inp["cu"][-1])
torch.manual_seed(11 + int(has_state))
# The layer hands the prologue a strided view of the QKVGFAB projection.
projection = torch.randn(total, 3 * lp + 64, device=dev, dtype=dtype) * 0.5
mixed = projection[:, : 3 * lp]
# conv1d is built with bias=False and fp32 weights; the fused kernel takes
# the width-major [qkv, width, channel] mirror the decode path already
# keeps.
weight = torch.randn(3, width, lp, device=dev, dtype=torch.float32) * 0.3
store = torch.zeros(n, width - 1, 3 * lp, device=dev, dtype=dtype)
if has_state:
store.normal_(0.0, 0.5)
state = store.transpose(-1, -2)
idx = torch.arange(n, device=dev, dtype=torch.int32)
has_init = torch.full((n,), has_state, device=dev, dtype=torch.bool)
# Reference conv, per sequence, in fp32 then rounded like a bf16 output.
conv_ref = torch.empty(total, 3 * lp, device=dev, dtype=dtype)
state_ref = torch.empty_like(store)
flat_w = weight.permute(0, 2, 1).reshape(3 * lp, width)
for i, (b, e) in enumerate(zip(inp["cu"][:-1].tolist(), inp["cu"][1:].tolist())):
pre = (
store[i].float()
if has_state
else torch.zeros(width - 1, 3 * lp, device=dev, dtype=torch.float32)
)
xs = torch.cat([pre, mixed[b:e].float()], 0)
acc = torch.zeros(e - b, 3 * lp, device=dev, dtype=torch.float32)
for w in range(width):
acc += xs[w : w + (e - b)] * flat_w[:, w].unsqueeze(0)
conv_ref[b:e] = F.silu(acc).to(dtype)
state_ref[i] = xs[-(width - 1) :].to(dtype)
def _bands(x, contiguous):
out = [
b.unsqueeze(0).unflatten(-1, (NUM_HEADS, HEAD_DIM))
for b in x.split(lp, dim=-1)
]
return [b.contiguous() for b in out] if contiguous else out
# Compare through both kernels rather than the prologue workspaces: `aqk`
# is deliberately left unwritten past each chunk's valid rows, so those
# elements are whatever the allocator last held.
scale = HEAD_DIM**-0.5
def _through_kernels(q, k, v, conv: dict | None):
ws = fused_kda_prologue(
q=q,
k=k,
v=v,
raw_g=inp["raw_g"],
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
dt_bias=inp["dt_bias"],
scale=scale,
lower_bound=LOWER_BOUND,
cu_seqlens=inp["cu"],
**(conv or {}),
)
return fused_kda_chunk(
qg=ws["qg"],
w=ws["w"],
u=ws["u"],
kg_t=ws["kg_t"],
aqk=ws["aqk"],
decay=ws["decay"],
out=torch.empty_like(ws["u"]),
scale=scale,
cu_seqlens=inp["cu"],
initial_state=inp["h0"].clone(),
output_final_state=True,
)
qr, kr, vr = _bands(conv_ref, True)
o_ref, s_ref = _through_kernels(qr, kr, vr, None)
got_state = state.clone()
qf, kf, vf = _bands(mixed, False)
o_got, s_got = _through_kernels(
qf,
kf,
vf,
dict(
conv_weight=weight,
conv_state=got_state,
conv_state_indices=idx,
conv_has_initial_state=has_init,
),
)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(s_got, s_ref, rtol=1e-3, atol=1e-3)
torch.testing.assert_close(
got_state.transpose(-1, -2).float(), state_ref.float(), rtol=0, atol=0
)
@pytest.mark.parametrize("groups", [2, 4, 8])
@pytest.mark.parametrize("seqlens", [[64], [64, 64], [100, 64], [513, 64, 1]])
def test_chunk_group_scan_matches_serial_walk(
groups: int, seqlens: list[int], monkeypatch
) -> None:
"""Splitting the chunk walk into parallel groups must not change the result.
Two bugs lived here, both invisible while the path was gated to >= 128
chunks per sequence:
* a group with no chunks returned before writing its ``B_g``, so the scan
folded whatever the workspace last held into the final state — these
shapes have fewer chunks than groups, so most groups are empty;
* the second pass started every group after the first from a zero state
instead of the one the scan composed, which only looked right because a
strongly decaying gate makes the dropped term small.
"""
_requires_kernel()
inp = _inputs(seqlens, seed=5 + groups)
_force_groups(monkeypatch, 1)
o_ref, state_ref = _run(inp, use_fused=True)
_force_groups(monkeypatch, groups)
o_got, state_got = _run(inp, use_fused=True)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize("seqlens", [[320], [513, 64, 1, 1200], [64, 64, 64]])
def test_accepts_chunk_metadata_from_the_attention_builder(
seqlens: list[int],
) -> None:
"""The layer passes precomputed chunk metadata, not None.
``KimiK3ROCmKDAMetadataBuilder`` builds these on device to avoid a blocking
device->host copy. Passing ``None`` instead takes a different path that
builds them locally, so drive the op with the builder's own tensors.
"""
_requires_kernel()
from vllm.models.kimi_k3.amd.kda_metadata import prepare_chunk_metadata_device
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
inp = _inputs(seqlens, seed=17)
cu = inp["cu"]
chunk_indices, chunk_offsets = prepare_chunk_metadata_device(
cu, cu.cpu(), FLA_CHUNK_SIZE
)
o_ref, state_ref = _run(inp, use_fused=True)
o_got, state_got = _run(
inp, use_fused=True, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets
)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
def _checkpoint_buffer(rows: int) -> torch.Tensor:
"""A destination filled with NaN, so an unwritten row is unmistakable."""
return torch.full(
(rows, NUM_HEADS, HEAD_DIM, HEAD_DIM),
float("nan"),
device="cuda",
dtype=torch.float32,
)
def _prefix_inputs(inp: dict, seq: int, num_tokens: int) -> dict:
"""Sequence ``seq`` truncated to its first ``num_tokens`` tokens."""
bos = int(inp["cu"][seq])
tokens = slice(bos, bos + num_tokens)
prefix = dict(inp)
for key in ("q", "k", "v", "raw_g", "raw_beta"):
prefix[key] = inp[key][:, tokens].contiguous()
prefix["h0"] = inp["h0"][seq : seq + 1].contiguous()
prefix["cu"] = torch.tensor([0, num_tokens], device="cuda", dtype=torch.int32)
return prefix
def _run_with_checkpoints(
inp: dict, offsets: list[int], rows: int | None = None, **kw
) -> tuple[torch.Tensor, tuple]:
ckpt = _checkpoint_buffer(len(offsets) if rows is None else rows)
result = _run(
inp,
use_fused=True,
checkpoint_state=ckpt,
checkpoint_offsets=torch.tensor(offsets, device="cuda", dtype=torch.int32),
**kw,
)
return ckpt, result
@pytest.mark.parametrize(
"seqlens,offsets",
[
([256], [64]), # the first interior boundary
([256], [192]), # the last one
([1024], [512]),
([320, 320], [64, 256]), # every sequence checkpoints
([513, 64, 1, 1200], [448, 0, 0, 640]), # ragged, most opted out
],
)
def test_checkpoint_matches_a_prefill_that_stops_at_the_offset(
seqlens: list[int], offsets: list[int], monkeypatch
) -> None:
"""A checkpoint is only useful if resuming from it is indistinguishable.
The scheduler serves the exported state on a later prefix-cache hit as if
the prompt had been prefilled up to that block and no further, so the
kernel has to hand back exactly what a prefill truncated there produces --
not merely something close. The walk passes through that state anyway, so
equality is bit-exact and the test asserts it as such.
Sequences with offset 0 opt out, and their rows must stay untouched.
"""
_requires_kernel()
_force_groups(monkeypatch, 1)
inp = _inputs(seqlens, seed=23 + len(seqlens))
ckpt, _ = _run_with_checkpoints(inp, offsets)
for i, offset in enumerate(offsets):
if offset == 0:
assert ckpt[i].isnan().all()
continue
_, truncated = _run(_prefix_inputs(inp, i, offset), use_fused=True)
torch.testing.assert_close(ckpt[i], truncated[0], rtol=0, atol=0)
def test_checkpoint_leaves_the_output_and_final_state_alone() -> None:
"""Exporting a checkpoint must not perturb the walk it is taken from."""
_requires_kernel()
seqlens = [768, 320]
inp = _inputs(seqlens, seed=29)
o_ref, state_ref = _run(inp, use_fused=True)
_, (o_got, state_got) = _run_with_checkpoints(inp, [512, 128])
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=0, atol=0)
torch.testing.assert_close(state_got, state_ref, rtol=0, atol=0)
@pytest.mark.parametrize("offset", [100, 63, 65, 1024])
def test_unusable_checkpoint_offsets_write_nothing(offset: int) -> None:
"""Offsets the walk never reaches must leave the destination untouched.
The recurrent state only exists at chunk boundaries, so an offset that is
not a multiple of ``KDA_CHECKPOINT_ALIGNMENT`` -- or that lies past the end
of the sequence -- cannot be honoured. Rounding it to a nearby boundary
would hand the scheduler a state for the wrong token count, so the kernel
stores nothing and leaves the caller to filter these out before the block
is hashed.
"""
_requires_kernel()
from vllm.models.kimi_k3.amd.ops.kda_chunk import KDA_CHECKPOINT_ALIGNMENT
assert offset % KDA_CHECKPOINT_ALIGNMENT != 0 or offset > 512
inp = _inputs([512], seed=31)
ckpt, _ = _run_with_checkpoints(inp, [offset])
assert ckpt.isnan().all()
def test_checkpoint_state_indices_pick_the_destination_row() -> None:
"""With indices the export lands straight in the paged state cache.
FlashKDA writes one row per sequence into a staging buffer that the caller
then scatters. The ROCm walk holds the state in registers already, so it
can be told which cache block each sequence owns and skip the staging
buffer and its scatter launch entirely. A negative row opts a sequence out,
the same way a zero offset does.
"""
_requires_kernel()
seqlens = [256, 256, 256]
rows = 8
inp = _inputs(seqlens, seed=37)
indices = [5, -1, 2]
ckpt, _ = _run_with_checkpoints(
inp,
[64, 64, 128],
rows=rows,
checkpoint_state_indices=torch.tensor(
indices, device="cuda", dtype=torch.int32
),
)
written = {5: (0, 64), 2: (2, 128)}
for row in range(rows):
if row not in written:
assert ckpt[row].isnan().all(), f"row {row} was written"
continue
seq, offset = written[row]
_, truncated = _run(_prefix_inputs(inp, seq, offset), use_fused=True)
torch.testing.assert_close(ckpt[row], truncated[0], rtol=0, atol=0)
@pytest.mark.parametrize("groups", [2, 4])
def test_checkpoint_survives_the_group_split(groups: int, monkeypatch) -> None:
"""The split walks each group from a composed state, not from zero.
Only the second pass holds the true state at an interior boundary, so a
checkpoint taken from the first pass would be the group-local state and
silently wrong.
"""
_requires_kernel()
inp = _inputs([1024], seed=41)
_force_groups(monkeypatch, 1)
_, serial = _run(_prefix_inputs(inp, 0, 512), use_fused=True)
_force_groups(monkeypatch, groups)
ckpt, _ = _run_with_checkpoints(inp, [512])
torch.testing.assert_close(ckpt[0], serial[0], rtol=1e-3, atol=1e-3)
def test_triton_fallback_refuses_a_checkpoint_export() -> None:
"""Dropping the export silently would serve an unwritten state.
The scheduler allocates and hashes the checkpoint block on the strength of
the layer opting in, so a backend that cannot write it has to say so rather
than return a plausible output.
"""
inp = _inputs([256], seed=43)
with pytest.raises(NotImplementedError):
_run(
inp,
use_fused=False,
checkpoint_state=_checkpoint_buffer(1),
checkpoint_offsets=torch.tensor([64], device="cuda", dtype=torch.int32),
)
# ---------------------------------------------------------------------------
# Paged recurrent state
#
# Each sequence reads its initial state from a row of the paged cache and
# writes its final state back to the same row. These check that the result
# matches the dense initial_state/final_state pair, that a sequence marked
# cold starts from zeros rather than from whatever its row held, and that rows
# no sequence owns are left untouched.
# ---------------------------------------------------------------------------
def _paged_cache(inp: dict, warm: list[bool], rows: list[int], slots: int):
"""A cache holding each sequence's h0 at its row, a fill pattern elsewhere."""
cache = torch.full(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
-7.0,
device="cuda",
dtype=torch.float32,
)
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
for n, row in enumerate(rows):
# A cold row keeps its fill pattern, which must not reach the walk.
if warm[n]:
cache[row] = inp["h0"][n]
return cache, idx, torch.tensor(warm, device="cuda", dtype=torch.bool)
def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
return chunk_kda_prefill(
q=inp["q"].clone(),
k=inp["k"].clone(),
v=inp["v"].clone(),
raw_g=inp["raw_g"].clone(),
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
g_bias=inp["dt_bias"],
lower_bound=LOWER_BOUND,
state_cache=cache,
state_indices=idx,
has_initial_state=warm,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
backend="fused" if use_fused else "triton",
**kw,
)
@pytest.mark.parametrize("use_fused", [False, True])
@pytest.mark.parametrize("seqlens", [[512], [2048], [320, 64], [513, 64, 1]])
def test_paged_state_matches_the_dense_pair(
use_fused: bool, seqlens: list[int]
) -> None:
"""A paged cache must give the same result as the dense pair."""
_requires_kernel()
inp = _inputs(seqlens, seed=11)
n = len(seqlens)
rows = [3 * i + 5 for i in range(n)]
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
before = cache.clone()
o_ref, ht_ref = _run(inp, use_fused=use_fused)
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
assert ht_got is None, "the final state is in the cache, not returned"
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
untouched = torch.ones(cache.shape[0], dtype=torch.bool, device="cuda")
untouched[idx.long()] = False
assert torch.equal(cache[untouched], before[untouched]), (
"a row no sequence owns was written"
)
@pytest.mark.parametrize("use_fused", [False, True])
def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
"""`has_initial_state=False` must ignore whatever the row already holds."""
_requires_kernel()
seqlens = [256, 256, 256]
inp = _inputs(seqlens, seed=12)
warm = [True, False, True]
rows = [4, 9, 1]
cache, idx, warm_t = _paged_cache(inp, warm, rows, slots=16)
# The dense reference zeroes exactly the cold sequences' initial states.
ref = dict(inp)
ref["h0"] = inp["h0"].clone()
ref["h0"][1] = 0.0
o_ref, ht_ref = _run(ref, use_fused=use_fused)
o_got, _ = _run_paged(inp, cache, idx, warm_t, use_fused=use_fused)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
@pytest.mark.parametrize("use_fused", [False, True])
def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None:
# The layer passes block_table[:, 0], which is contiguous only while the
# table has one column. Speculative decoding reserves a state page per
# draft token, so the table widens and that column becomes strided.
_requires_kernel()
seqlens = [512, 320, 64]
inp = _inputs(seqlens, seed=43)
n = len(seqlens)
rows = [3 * i + 5 for i in range(n)]
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
ref_cache = cache.clone()
table = torch.zeros(n, 9, dtype=torch.int32, device="cuda")
table[:, 0] = idx
column = table[:, 0]
assert not column.is_contiguous()
o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused)
o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused)
torch.testing.assert_close(o_got, o_ref)
torch.testing.assert_close(cache, ref_cache)
@pytest.mark.parametrize("use_fused", [False, True])
def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None:
# A batch carrying speculative decodes pads its non-spec group with
# zero-length sequences whose cache row still belongs to a request, so
# consuming no tokens has to mean writing nothing and perturbing nobody.
_requires_kernel()
live = [200, 1, 128]
# _inputs draws h0 last, so the padded batch's live rows are the same draw
# as the unpadded one's and the two runs compare elementwise.
padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41)
rows = [3 * i + 5 for i in range(4)]
cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24)
before = cache.clone()
_run_paged(padded, cache, idx, warm, use_fused=use_fused)
ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24)
_run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused)
assert torch.equal(cache[rows[3]], before[rows[3]]), (
"the padded sequence's cache row was written"
)
torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()])
@pytest.mark.parametrize("groups", [2, 4])
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
"""At G>1 the read and the write are different blocks of one launch."""
_requires_kernel()
_force_groups(monkeypatch, groups)
inp = _inputs([1024, 512], seed=13)
rows = [6, 2]
cache, idx, warm = _paged_cache(inp, [True, True], rows, slots=12)
o_ref, ht_ref = _run(inp, use_fused=True)
o_got, _ = _run_paged(inp, cache, idx, warm, use_fused=True)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
def test_paged_state_and_checkpoint_export_together() -> None:
"""The checkpoint lands in its own row while the walk owns the state row."""
_requires_kernel()
inp = _inputs([1024], seed=14)
cache, idx, warm = _paged_cache(inp, [True], [3], slots=12)
ckidx = torch.tensor([8], device="cuda", dtype=torch.int32)
o_ref, ht_ref = _run(inp, use_fused=True)
prefix = _prefix_inputs(inp, 0, 512)
_, ck_ref = _run(prefix, use_fused=True)
o_got, _ = _run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
checkpoint_state=cache,
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
checkpoint_state_indices=ckidx,
)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[ckidx.long()], ck_ref, rtol=5e-3, atol=5e-3)
def test_paged_state_rejects_a_dense_initial_state() -> None:
"""The two ways of passing the state are mutually exclusive."""
_requires_kernel()
inp = _inputs([256], seed=15)
cache, idx, warm = _paged_cache(inp, [True], [1], slots=4)
with pytest.raises(ValueError, match="replaces initial_state"):
_run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
initial_state=inp["h0"].clone(),
)
@pytest.mark.parametrize("use_fused", [False, True])
def test_padded_cache_stride_is_honoured(use_fused: bool) -> None:
"""A mamba page may pad stride(0); only each state row is dense.
The real `recurrent_state` is not a contiguous tensor, so a kernel that
assumes a packed ``H * V * K`` row stride addresses the wrong rows. The
other cases in this file all build a contiguous cache, which cannot
exercise that.
"""
_requires_kernel()
seqlens = [1024, 512]
inp = _inputs(seqlens, seed=21)
n = len(seqlens)
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
slots, pad = 16, 512
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
cache = raw.as_strided(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
)
assert not cache.is_contiguous() and cache[0].is_contiguous()
rows = [11, 3]
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
warm = torch.ones(n, dtype=torch.bool, device="cuda")
for i, r in enumerate(rows):
cache[r] = inp["h0"][i]
before = raw.clone()
o_ref, ht_ref = _run(inp, use_fused=use_fused)
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
assert ht_got is None
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
# The padding between rows, and every unowned row, must be untouched.
untouched = torch.ones(slots, dtype=torch.bool, device="cuda")
untouched[idx.long()] = False
assert torch.equal(raw[untouched], before[untouched])
assert torch.equal(raw[idx.long(), row:], before[idx.long(), row:]), (
"the kernel wrote into the inter-row padding"
)
def test_padded_checkpoint_stride_is_honoured() -> None:
"""The checkpoint destination is the paged cache, which pads stride(0).
Every other checkpoint case here builds a contiguous buffer, so none of
them would catch a kernel that assumes a packed ``H * V * K`` row stride
and writes the snapshot into a neighbouring block.
"""
_requires_kernel()
seqlens = [1024]
inp = _inputs(seqlens, seed=31)
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
slots, pad = 16, 512
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
cache = raw.as_strided(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
)
assert not cache.is_contiguous()
state_rows = [11]
idx = torch.tensor(state_rows, device="cuda", dtype=torch.int32)
warm = torch.ones(1, dtype=torch.bool, device="cuda")
cache[state_rows[0]] = inp["h0"][0]
ck_rows = [4]
before = raw.clone()
prefix = _prefix_inputs(inp, 0, 512)
_, ck_ref = _run(prefix, use_fused=True)
o_ref, ht_ref = _run(inp, use_fused=True)
o_got, _ = _run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
checkpoint_state=cache,
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
checkpoint_state_indices=torch.tensor(
ck_rows, device="cuda", dtype=torch.int32
),
)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[ck_rows[0]], ck_ref[0], rtol=5e-3, atol=5e-3)
touched = torch.zeros(slots, dtype=torch.bool, device="cuda")
touched[state_rows + ck_rows] = True
assert torch.equal(raw[~touched], before[~touched])
assert torch.equal(raw[touched, row:], before[touched, row:]), (
"the kernel wrote into the inter-row padding"
)
-659
View File
@@ -1,659 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
Two fused paths fold the packed causal conv1d update, the gated delta-rule
recurrence and the gated output RMSNorm into one launch, updating the conv
state and the recurrent state in place:
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
Every output of each is compared against the unfused path the AMD layer falls
back to, with the argument layout the layer itself uses.
"""
import pytest
import torch
from vllm.platforms import current_platform
def _hip_kernel_arch() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx942, on_gfx950
return on_gfx950() or on_gfx942()
def _aiter_gluon_kda_available() -> bool:
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.platforms.rocm import get_cdna_version
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
)
requires_hip_kernel_arch = pytest.mark.skipif(
not _hip_kernel_arch(),
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
)
requires_aiter_gluon_kda = pytest.mark.skipif(
not _aiter_gluon_kda_available(),
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
)
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
HEAD_DIM = 128
CONV_WIDTH = 4
GATE_LOWER_BOUND = -5.0
NORM_EPS = 1e-5
DTYPE = torch.bfloat16
def _requires_kernel() -> None:
if not hasattr(torch.ops._C, "fused_kda_decode"):
pytest.skip("vLLM was built without the fused KDA decode kernel")
class KdaDecodeInputs:
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
``num_spec`` > 0 sizes the conv state for speculative decoding
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
"""
def __init__(
self,
num_tokens: int,
num_heads: int,
num_slots: int,
seed: int = 0,
num_spec: int = 0,
) -> None:
torch.manual_seed(seed)
device = "cuda"
dim = num_heads * HEAD_DIM
self.num_heads = num_heads
self.dim = dim
self.mixed_qkv = (
torch.randn(num_tokens, 3 * dim, device=device, dtype=DTYPE) * 0.5
)
# conv1d weight as the layer holds it: [3 * dim, width] fp32.
self.conv_weights = (
torch.randn(3 * dim, CONV_WIDTH, device=device, dtype=torch.float32) * 0.3
)
# Width-major fp32 copy the fused kernel indexes: [3, width, dim].
self.decode_conv1d_weight = torch.stack(
[
self.conv_weights[i * dim : (i + 1) * dim].transpose(0, 1).contiguous()
for i in range(3)
]
)
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
self.conv_state = (
torch.randn(
num_slots,
CONV_WIDTH - 1 + num_spec,
3 * dim,
device=device,
dtype=DTYPE,
)
* 0.5
)
self.recurrent_state = (
torch.randn(
num_slots,
num_heads,
HEAD_DIM,
HEAD_DIM,
device=device,
dtype=torch.float32,
)
* 0.1
)
self.g1 = (
torch.randn(1, num_tokens, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
* 0.5
)
self.g2 = (
torch.randn(num_tokens, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
* 0.5
)
self.beta = torch.randn(1, num_tokens, num_heads, device=device, dtype=DTYPE)
self.A_log = torch.randn(num_heads, device=device, dtype=torch.float32) * 0.5
self.dt_bias = torch.randn(dim, device=device, dtype=torch.float32) * 0.1
self.norm_weight_bf16 = 1 + 0.1 * torch.randn(
HEAD_DIM, device=device, dtype=DTYPE
)
self.decode_norm_weight = self.norm_weight_bf16.float()
# Distinct, shuffled slots: the kernel must honour the indirection.
# Slot 0 is NULL_BLOCK_ID and never allocated for live state, and the
# Triton reference short-circuits it, so draw from [1, num_slots).
assert num_slots > num_tokens, "need a spare slot to detect stray writes"
self.state_indices = (
torch.randperm(num_slots - 1, device=device)[:num_tokens] + 1
).to(torch.int32)
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
return state.transpose(-1, -2)
def empty_out(self) -> torch.Tensor:
num_tokens = self.mixed_qkv.shape[0]
return torch.empty(
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
)
def _gated_rmsnorm(
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
) -> torch.Tensor:
x_float = x.float()
variance = x_float.pow(2).mean(dim=-1, keepdim=True)
normed = x_float * torch.rsqrt(variance + eps) * weight.float()
return (normed * torch.sigmoid(gate.float())).to(x.dtype)
def _run_triton_chain(
inp: KdaDecodeInputs,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""conv1d update -> recurrent decode -> gated norm, as the layer runs it."""
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
fused_recurrent_kda_packed_decode,
)
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
conv_out = torch.empty_like(inp.mixed_qkv)
causal_conv1d_update(
inp.mixed_qkv,
inp.conv_state_view(conv_state),
inp.conv_weights,
None,
activation="silu",
conv_state_indices=inp.state_indices,
validate_data=True,
out=conv_out,
)
core_attn_out, _ = fused_recurrent_kda_packed_decode(
mixed_qkv=conv_out,
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
state_indices=inp.state_indices,
)
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
return out, conv_state, recurrent_state
def _run_fused(
inp: KdaDecodeInputs,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
from vllm import _custom_ops as ops
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
num_tokens = inp.mixed_qkv.shape[0]
out = torch.empty(
1,
num_tokens,
inp.num_heads,
HEAD_DIM,
device=inp.mixed_qkv.device,
dtype=DTYPE,
)
ops.fused_kda_decode(
x=inp.mixed_qkv,
weight=inp.decode_conv1d_weight,
bias=None,
conv_state=inp.conv_state_view(conv_state),
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
state_indices=inp.state_indices,
state=recurrent_state,
out=out,
lower_bound=GATE_LOWER_BOUND,
output_gate=inp.g2,
norm_weight=inp.decode_norm_weight,
norm_eps=NORM_EPS,
)
return out, conv_state, recurrent_state
@requires_hip_kernel_arch
@pytest.mark.parametrize("num_heads", [12, 24, 96])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int) -> None:
_requires_kernel()
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_fused(inp)
# The fused kernel keeps the recurrent output in fp32 through the norm,
# while the Triton chain rounds it to BF16 in between.
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
# The conv state is a pure shift-and-append: it must be bit-exact.
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by state_indices may be written."""
_requires_kernel()
num_tokens, num_heads, num_slots = 4, 12, 9
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
_, actual_conv, actual_state = _run_fused(inp)
touched = set(inp.state_indices.tolist())
untouched = [slot for slot in range(num_slots) if slot not in touched]
assert untouched, "test needs at least one unused slot"
torch.testing.assert_close(
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
)
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_without_output_norm() -> None:
"""Omitting the gate/norm pair returns the raw recurrent output."""
_requires_kernel()
from vllm import _custom_ops as ops
inp = KdaDecodeInputs(5, 12, num_slots=8, seed=7)
_, _, expected_state = _run_triton_chain(inp)
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
out = torch.empty(1, 5, inp.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE)
ops.fused_kda_decode(
x=inp.mixed_qkv,
weight=inp.decode_conv1d_weight,
bias=None,
conv_state=inp.conv_state_view(conv_state),
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
state_indices=inp.state_indices,
state=recurrent_state,
out=out,
lower_bound=GATE_LOWER_BOUND,
)
torch.testing.assert_close(recurrent_state, expected_state, atol=2e-3, rtol=2e-3)
assert torch.isfinite(out).all()
@requires_hip_kernel_arch
@torch.inference_mode()
def test_fused_kda_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0, and
the Triton chain zeroes those rows' output while leaving slot 0 alone. The
fused kernel must do the same rather than read-modify-write slot 0's state
once per padded row.
"""
_requires_kernel()
num_real, num_padded, num_heads = 3, 5, 12
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
inp.state_indices[num_real:] = 0
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_fused(inp)
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
assert not actual_out[0, num_real:].any(), "padded rows must produce zeros"
# Slot 0 must be untouched by both paths.
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_hip_kernel_arch
def test_decode_weights_staged_after_loading(dist_init) -> None:
"""The fused-decode weight copies must not depend on the weight loaders.
Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left
them uninitialized, so decode could read NaN. Loading runs with grad mode
on, so the copies must also stay out of autograd.
"""
_requires_kernel()
from tests.v1.attention.utils import create_vllm_config
from vllm.config import set_current_vllm_config
from vllm.model_executor.model_loader.utils import process_weights_after_loading
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
config = KimiLinearConfig(
hidden_size=256,
linear_attn_config={
"kda_layers": [1],
"full_attn_layers": [],
"head_dim": HEAD_DIM,
"num_heads": 12,
"short_conv_kernel_size": CONV_WIDTH,
"use_full_rank_gate": True,
"gate_lower_bound": GATE_LOWER_BOUND,
},
)
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE)
with set_current_vllm_config(vllm_config), torch.device("cuda"):
layer = KimiK3DeltaAttention(
config, vllm_config, prefix="model.layers.0.self_attn"
)
assert layer.decode_conv1d_weight is not None
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
# kernel gets an fp32 norm copy.
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
with torch.no_grad():
for param in layer.parameters():
param.normal_()
layer.decode_conv1d_weight.fill_(float("nan"))
if layer.decode_norm_weight is not None:
layer.decode_norm_weight.fill_(float("nan"))
process_weights_after_loading(
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
)
dim = layer.local_projection_size
conv_weights = layer.conv1d.weight.squeeze(1)
expected_conv = torch.stack(
[conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)]
)
torch.testing.assert_close(
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
)
if layer.decode_norm_weight is not None:
torch.testing.assert_close(
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
)
assert not layer.decode_norm_weight.requires_grad
assert not layer.decode_conv1d_weight.requires_grad
# --------------------------------------------------------------------------
# non-spec decode: fused_recurrent_kda_packed_decode
# --------------------------------------------------------------------------
def _run_aiter_packed_decode(
inp: KdaDecodeInputs,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
from vllm._aiter_ops import rocm_aiter_ops
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
num_tokens = inp.mixed_qkv.shape[0]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
mixed_qkv=inp.mixed_qkv,
g=inp.g1[0],
beta=inp.beta[0],
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
ssm_state_indices=inp.state_indices[:num_tokens],
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:num_tokens],
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[0, :num_tokens],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
@torch.inference_mode()
def test_aiter_packed_decode_matches_triton_chain(
num_heads: int, num_tokens: int
) -> None:
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
# The fused kernel keeps the recurrent output in fp32 through the norm,
# while the Triton chain rounds it to BF16 in between.
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
# The conv state is a pure shift-and-append: it must be bit-exact.
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
"""Only the slots named by the state indices may be written."""
num_tokens, num_heads, num_slots = 4, 12, 9
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
touched = set(inp.state_indices.tolist())
untouched = [slot for slot in range(num_slots) if slot not in touched]
assert untouched, "test needs at least one unused slot"
torch.testing.assert_close(
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
)
@requires_aiter_gluon_kda
@torch.inference_mode()
def test_aiter_packed_decode_skips_null_block_padding() -> None:
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
must do the same rather than read-modify-write slot 0 once per padded row.
"""
num_real, num_padded, num_heads = 3, 5, 12
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
inp.state_indices[num_real:] = 0
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
torch.testing.assert_close(
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
)
assert torch.isfinite(actual_out).all()
# Slot 0 must be untouched by both paths.
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
# --------------------------------------------------------------------------
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
# --------------------------------------------------------------------------
class SpecBatch:
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
per-token slot table the GDN metadata hands the layer.
"""
def __init__(
self,
seq_lens: list[int],
num_accepted: list[int],
num_heads: int,
num_spec: int,
seed: int = 0,
) -> None:
assert len(seq_lens) == len(num_accepted)
assert max(seq_lens) <= num_spec + 1
n_seqs = len(seq_lens)
slots_per_seq = num_spec + 1
num_slots = n_seqs * slots_per_seq + 2
self.inp = KdaDecodeInputs(
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
)
# Distinct slots per (request, token), shuffled, none at slot 0.
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
self.query_start_loc = torch.tensor(
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
dtype=torch.int32,
device="cuda",
)
self.num_accepted_tokens = torch.tensor(
num_accepted, dtype=torch.int32, device="cuda"
)
self.max_query_len = slots_per_seq
def _run_spec_triton_chain(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
conv_out = causal_conv1d_update(
inp.mixed_qkv,
inp.conv_state_view(conv_state),
inp.conv_weights,
None,
activation="silu",
conv_state_indices=sb.state_indices[:, 0],
num_accepted_tokens=sb.num_accepted_tokens,
query_start_loc=sb.query_start_loc,
max_query_len=sb.max_query_len,
validate_data=False,
out=torch.empty_like(inp.mixed_qkv),
)
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
)
core_attn_out, _ = fused_recurrent_kda(
q=q,
k=k,
v=v,
raw_g=inp.g1,
raw_beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
)
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
return out, conv_state, recurrent_state
def _run_aiter_spec(
sb: SpecBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
from vllm._aiter_ops import rocm_aiter_ops
inp = sb.inp
conv_state = inp.conv_state.clone()
recurrent_state = inp.recurrent_state.clone()
q, k, v = (
x.view(1, -1, inp.num_heads, HEAD_DIM)
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
)
n_spec = q.shape[1]
core_attn_out = inp.empty_out()
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
q=q,
k=k,
v=v,
g=inp.g1,
beta=inp.beta,
A_log=inp.A_log,
dt_bias=inp.dt_bias,
lower_bound=GATE_LOWER_BOUND,
initial_state=recurrent_state,
cu_seqlens=sb.query_start_loc,
ssm_state_indices=sb.state_indices,
num_accepted_tokens=sb.num_accepted_tokens,
conv_state=inp.conv_state_view(conv_state),
conv_weight=inp.decode_conv1d_weight,
out_gate=inp.g2[:n_spec].unsqueeze(0),
norm_weight=inp.norm_weight_bf16,
norm_eps=NORM_EPS,
out=core_attn_out[:, :n_spec],
)
return core_attn_out, conv_state, recurrent_state
@requires_aiter_gluon_kda
@pytest.mark.parametrize("num_heads", [12, 24])
@pytest.mark.parametrize(
"num_spec, seq_lens, num_accepted",
[
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
(6, [7, 3], [1, 4]), # longer window, two requests
],
)
@torch.inference_mode()
def test_aiter_spec_decode_matches_triton_chain(
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
) -> None:
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
# Nothing outside the slot table may change.
touched = set(sb.state_indices.flatten().tolist())
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
assert untouched
torch.testing.assert_close(
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
)
torch.testing.assert_close(
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
)
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import torch
from torch import nn
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.amd.linear import KimiDecoderLayer
class _ReturningAttention(nn.Module):
def __init__(self, result: torch.Tensor) -> None:
super().__init__()
self.result = result
def forward(
self, hidden_states: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor:
return self.result
class _TupleProjection(nn.Module):
def __init__(self, result: torch.Tensor) -> None:
super().__init__()
self.result = result
def forward(self, _: torch.Tensor) -> tuple[torch.Tensor, None]:
return self.result, None
def _make_layer(self_attn: nn.Module) -> KimiDecoderLayer:
layer = object.__new__(KimiDecoderLayer)
nn.Module.__init__(layer)
layer.self_attn = self_attn
return layer
def test_kda_self_attention_returns_projection_storage_directly() -> None:
hidden_states = torch.randn(4, 8)
projected = torch.randn_like(hidden_states)
layer = _make_layer(_ReturningAttention(projected))
output = layer._run_self_attn(torch.arange(4), hidden_states)
assert output is projected
assert output.data_ptr() == projected.data_ptr()
def test_k3_kda_forward_returns_output_projection_storage() -> None:
hidden_states = torch.randn(4, 8)
projected = torch.randn_like(hidden_states)
def fill_core_output(**kwargs: torch.Tensor) -> None:
kwargs["core_attn_out"].fill_(1)
attention = SimpleNamespace(
local_projection_size=4,
head_dim=2,
local_num_heads=2,
in_proj_padding=0,
in_proj_qkvgfab=_TupleProjection(torch.randn(4, 20)),
f_b_proj=_TupleProjection(torch.randn(4, 4)),
o_proj=_TupleProjection(projected),
_forward=fill_core_output,
)
output = KimiK3DeltaAttention.forward(attention, hidden_states, torch.arange(4))
assert output is projected
assert output.data_ptr() == projected.data_ptr()
@@ -1,163 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="AITER fused conv1d is ROCm-only"
)
WIDTH = 4 # Kimi-K3 short_conv_kernel_size
STATE_LEN = WIDTH - 1
HEAD_DIM = 128
LOCAL_HEADS = 4 # per-TP-rank heads; projection = 512, packed dim = 1536
def _fused_fn():
gdn = pytest.importorskip("aiter.ops.triton.gated_delta_net")
return gdn.causal_conv1d_split_qkv_triton_tile_fn
def _make_inputs(seqlens, has_initial_state, seed=0, device="cuda"):
torch.manual_seed(seed)
proj = HEAD_DIM * LOCAL_HEADS
dim = 3 * proj
total = sum(seqlens)
n_seqs = len(seqlens)
# Token-major activations, as produced by the in_proj GEMM.
mixed_qkv = torch.randn(total, dim, dtype=torch.bfloat16, device=device)
weight = torch.randn(dim, WIDTH, dtype=torch.bfloat16, device=device) * 0.3
# More cache lines than sequences, assigned in a shuffled order. Line 0 is
# vLLM's null block (NULL_BLOCK_ID), which causal_conv1d_fn skips, so real
# sequences never map to it.
n_lines = n_seqs + 2
conv_state = torch.randn(
n_lines, dim, STATE_LEN, dtype=torch.bfloat16, device=device
)
cache_indices = (torch.randperm(n_lines - 1, device=device)[:n_seqs] + 1).to(
torch.int32
)
query_start_loc = torch.tensor(
[0] + list(torch.tensor(seqlens).cumsum(0).tolist()),
dtype=torch.int32,
device=device,
)
has_initial = torch.full(
(n_seqs,), has_initial_state, dtype=torch.bool, device=device
)
return mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
def _reference(
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
):
"""fp32 per-sequence depthwise causal conv + SiLU.
Returns (out[T, dim], new_conv_state).
"""
x = mixed_qkv.float().t() # (dim, T)
w = weight.float()
dim = x.shape[0]
out = torch.empty_like(x)
new_state = conv_state.float().clone()
qsl = query_start_loc.tolist()
for s in range(len(qsl) - 1):
start, end = qsl[s], qsl[s + 1]
ci = int(cache_indices[s])
xs = x[:, start:end]
if bool(has_initial[s]):
xin = torch.cat([conv_state[ci].float(), xs], dim=1)
else:
xin = F.pad(xs, (STATE_LEN, 0))
y = F.conv1d(xin.unsqueeze(0), w.unsqueeze(1), None, groups=dim)[0]
out[:, start:end] = F.silu(y)
new_state[ci] = xin[:, -STATE_LEN:]
return out.t(), new_state
def _three_launch(
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
):
"""The pre-fusion KDA prefill path: one causal_conv1d_fn per projection."""
proj = mixed_qkv.shape[1] // 3
outs = []
for x, state, w in zip(
mixed_qkv.split(proj, dim=-1),
conv_state.split(proj, dim=-2),
weight.split(proj, dim=0),
):
outs.append(
causal_conv1d_fn(
x.transpose(0, 1),
w,
None,
activation="silu",
conv_states=state,
has_initial_state=has_initial,
cache_indices=cache_indices,
query_start_loc=query_start_loc,
).transpose(0, 1)
)
return outs
@pytest.mark.parametrize(
"seqlens", [[1024], [8, 1, 300], [64, 64, 64], [513, 64, 1, 1200]]
)
@pytest.mark.parametrize("has_initial_state", [False, True])
@pytest.mark.parametrize("block_m", [8, 128])
def test_fused_split_qkv_conv1d_matches_reference(seqlens, has_initial_state, block_m):
fused_fn = _fused_fn()
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial = _make_inputs(
seqlens, has_initial_state
)
proj = mixed_qkv.shape[1] // 3
ref_out, ref_state = _reference(
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial
)
state_3 = conv_state.clone()
q3, k3, v3 = _three_launch(
mixed_qkv, weight, state_3, cache_indices, qsl, has_initial
)
state_f = conv_state.clone()
# Exactly the call made by KimiK3DeltaAttention._forward.
qf, kf, vf = fused_fn(
mixed_qkv.transpose(0, 1),
weight,
None,
state_f,
qsl,
proj,
proj,
cache_indices=cache_indices,
has_initial_state=has_initial,
activation="silu",
block_m=block_m,
metadata=None,
preserve_input_dtype=True,
)
fused = torch.cat([qf, kf, vf], dim=-1)
assert fused.shape == mixed_qkv.shape and fused.dtype == mixed_qkv.dtype
assert all(t.is_contiguous() for t in (qf, kf, vf))
tol = dict(atol=2e-2, rtol=2e-2)
torch.testing.assert_close(fused.float(), ref_out, **tol)
torch.testing.assert_close(fused, torch.cat([q3, k3, v3], dim=-1), **tol)
# conv_state writeback: assigned lines updated, the others untouched.
torch.testing.assert_close(state_f.float(), ref_state, **tol)
torch.testing.assert_close(state_f, state_3, **tol)
untouched = torch.ones(conv_state.shape[0], dtype=torch.bool, device="cuda")
untouched[cache_indices.long()] = False
assert untouched[NULL_BLOCK_ID]
assert torch.equal(state_f[untouched], conv_state[untouched])
@@ -1,441 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm latent-MoE tail must equal the replicated up-projection.
A wrong shard offset or a dropped accumulation still runs and still produces
plausible text, so these pin the arithmetic rather than the behaviour.
"""
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.nn.functional as F
from torch.multiprocessing import spawn
from tests.utils import (
ensure_current_vllm_config,
init_test_distributed_environment,
multi_gpu_test,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.runner import moe_runner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear
from vllm.models.kimi_k3.amd import latent_moe_runner
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCmLatentMoERunner is only wired up on ROCm",
)
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
EPS = 1e-5
DTYPE = torch.bfloat16
def _build_transform(
device: torch.device, *, row_sharded: bool = False
) -> tuple[KimiRoutedOutputTransform, torch.Tensor]:
norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE)
torch.manual_seed(0)
norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight))
full_weight = (
torch.randn(HIDDEN_SIZE, LATENT_SIZE, device=device, dtype=DTYPE)
/ LATENT_SIZE**0.5
)
up_proj: ReplicatedLinear | ColumnParallelLinear
if row_sharded:
up_proj = ColumnParallelLinear(
LATENT_SIZE,
HIDDEN_SIZE,
bias=False,
gather_output=True,
params_dtype=DTYPE,
prefix="routed_expert_up_proj",
).to(device=device)
up_proj.weight_loader(up_proj.weight, full_weight)
else:
up_proj = ReplicatedLinear(
LATENT_SIZE,
HIDDEN_SIZE,
bias=False,
params_dtype=DTYPE,
prefix="routed_expert_up_proj",
).to(device=device)
up_proj.weight.data.copy_(full_weight)
return (
KimiRoutedOutputTransform(norm, up_proj, row_sharded=row_sharded),
full_weight,
)
def _tail_runner(
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
) -> ROCmLatentMoERunner:
"""A runner carrying only what the tail reads, so no engine is needed.
``_maybe_reduce_final_output`` is left as the real base-class method, so the
all-reduce that stitches the shards is the real collective.
``tp_world`` is the size of the TP process group, which is what the hidden
dim is split by. Under expert parallelism the MoE config reports
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
that the final reduce still fires.
"""
runner = object.__new__(ROCmLatentMoERunner)
attrs = {
"routed_output_transform": transform,
"_up_proj_preshard": bool(getattr(transform, "row_sharded", False)),
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
"_logged_sharded_tail": False,
"moe_config": SimpleNamespace(
tp_size=1 if use_ep else tp_world,
ep_size=tp_world if use_ep else 1,
is_sequence_parallel=False,
skip_final_all_reduce=False,
),
}
for name, value in attrs.items():
object.__setattr__(runner, name, value)
return runner
def _all_reduced(tensor: torch.Tensor, group) -> torch.Tensor:
reduced = tensor.clone()
dist.all_reduce(reduced, group=group)
return reduced
def _rank_partials(
num_tokens: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor]:
routed = torch.randn(num_tokens, LATENT_SIZE, device=device, dtype=DTYPE)
shared = torch.randn(num_tokens, HIDDEN_SIZE, device=device, dtype=DTYPE)
return routed.mul_(0.01), shared
def _check_matches_replicated(
device: torch.device,
tp_world: int,
rank: int,
use_ep: bool = False,
row_sharded: bool = False,
) -> None:
transform, up_weight = _build_transform(device, row_sharded=row_sharded)
runner = _tail_runner(transform, tp_world, use_ep)
group = get_tp_group().device_group
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
torch.manual_seed(100 * iteration + rank + 1)
routed_output, shared_output = _rank_partials(num_tokens, device)
expected = F.linear(
F.rms_norm(
_all_reduced(routed_output, group),
(LATENT_SIZE,),
transform.norm.weight,
EPS,
),
up_weight,
)
expected.add_(_all_reduced(shared_output, group))
actual = runner._shard_up_proj_tail(routed_output, shared_output, None)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def _check_matches_replicated_under_ep(
device: torch.device, tp_world: int, rank: int
) -> None:
"""The served config: the routed experts are expert-parallel, so the MoE
config reports no tensor parallelism at all for a group that still splits
the hidden dim ``tp_world`` ways."""
_check_matches_replicated(device, tp_world, rank, use_ep=True)
def _check_presharded_matches_replicated(
device: torch.device, tp_world: int, rank: int
) -> None:
_check_matches_replicated(device, tp_world, rank, row_sharded=True)
def _check_unfused_tail_gathers_full_hidden(
device: torch.device, tp_world: int, rank: int
) -> None:
transform, up_weight = _build_transform(device, row_sharded=True)
assert transform.up_proj.weight.shape[0] == HIDDEN_SIZE // tp_world
# all-gather reconstructs the full projection when all ranks feed the same
# input, so use one seed for all ranks.
torch.manual_seed(1234)
latent = torch.randn(4, LATENT_SIZE, device=device, dtype=DTYPE).mul_(0.01)
torch.testing.assert_close(
transform(latent),
F.linear(transform.norm(latent), up_weight),
atol=8e-2,
rtol=3e-2,
)
def _check_writes_only_its_own_shard(
device: torch.device, tp_world: int, rank: int
) -> None:
"""Two ranks that swapped both weight rows and write offsets still sum to
the right total, so inspect each slice before the final collective."""
transform, up_weight = _build_transform(device)
runner = _tail_runner(transform, tp_world)
group = get_tp_group().device_group
torch.manual_seed(rank + 1)
routed_output, shared_output = _rank_partials(8, device)
before = shared_output.clone()
latent = transform.norm(_all_reduced(routed_output, group))
captured: dict = {}
real_reduce = runner._maybe_reduce_final_output
def _capture(states, trunc_size, output_is_reduced=None):
captured["states"] = states.clone()
captured["output_is_reduced"] = output_is_reduced
return real_reduce(states, trunc_size, output_is_reduced)
object.__setattr__(runner, "_maybe_reduce_final_output", _capture)
runner._shard_up_proj_tail(routed_output, shared_output, None)
# Restoring the method breaks the runner -> _capture -> runner cycle, so the
# captured device tensor is freed before the process group is torn down.
object.__setattr__(runner, "_maybe_reduce_final_output", real_reduce)
assert captured["output_is_reduced"] is False
shard = HIDDEN_SIZE // tp_world
start, end = rank * shard, (rank + 1) * shard
local = captured["states"]
projected = F.linear(latent, up_weight[start:end])
torch.testing.assert_close(
local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2
)
torch.testing.assert_close(local[:, :start], before[:, :start], atol=0, rtol=0)
torch.testing.assert_close(local[:, end:], before[:, end:], atol=0, rtol=0)
_CHECKS = {
"matches_replicated": _check_matches_replicated,
"matches_replicated_ep": _check_matches_replicated_under_ep,
"presharded_matches_replicated": _check_presharded_matches_replicated,
"unfused_gathers": _check_unfused_tail_gathers_full_hidden,
"own_shard_only": _check_writes_only_its_own_shard,
}
def _worker(local_rank: int, world_size: int, port: str, check: str) -> None:
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
with ensure_current_vllm_config():
init_test_distributed_environment(
world_size, 1, local_rank, port, local_rank=local_rank
)
_CHECKS[check](device, world_size, local_rank)
def _run_ranks(check: str, tp_size: int) -> None:
spawn(
_worker,
args=(tp_size, str(get_open_port()), check),
nprocs=tp_size,
join=True,
)
@multi_gpu_test(num_gpus=4)
def test_sharded_tail_tp4_matches_replicated_projection() -> None:
_run_ranks("matches_replicated", 4)
@multi_gpu_test(num_gpus=8)
def test_sharded_tail_tp8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated", 8)
@multi_gpu_test(num_gpus=8)
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
_run_ranks("matches_replicated_ep", 8)
@multi_gpu_test(num_gpus=4)
def test_presharded_tail_tp4_matches_replicated_projection() -> None:
_run_ranks("presharded_matches_replicated", 4)
@multi_gpu_test(num_gpus=4)
def test_presharded_unfused_tail_gathers_the_full_hidden_dim() -> None:
_run_ranks("unfused_gathers", 4)
@multi_gpu_test(num_gpus=4)
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
_run_ranks("own_shard_only", 4)
def _runner(**attrs) -> ROCmLatentMoERunner:
runner = object.__new__(ROCmLatentMoERunner)
for name, value in attrs.items():
object.__setattr__(runner, name, value)
return runner
@pytest.mark.parametrize(
"shardable,pre_reduced,expect_sharded",
[
(True, False, True),
(True, True, False),
(False, False, False),
(False, True, False),
],
)
def test_forward_shards_only_when_the_tail_is_valid(
shardable: bool,
pre_reduced: bool,
expect_sharded: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The shard needs an un-reduced shared partial to accumulate into."""
base_output, sharded_output = torch.zeros(1), torch.ones(1)
monkeypatch.setattr(moe_runner.MoERunner, "forward", lambda *a, **k: base_output)
monkeypatch.setattr(
ROCmLatentMoERunner, "_fused_forward", lambda *a, **k: sharded_output
)
monkeypatch.setattr(
ROCmLatentMoERunner, "_fused_output_is_reduced", property(lambda _: pre_reduced)
)
runner = _runner(_tail_shardable=shardable)
result = runner.forward(torch.zeros(1), torch.zeros(1))
assert result is (sharded_output if expect_sharded else base_output)
@pytest.fixture
def build_runner(monkeypatch: pytest.MonkeyPatch):
"""Construct through the real subclass ``__init__``, stubbing only the base.
``tp_world`` is the size of the TP process group and is varied independently
of ``moe_config.tp_size``, because under expert parallelism those two are
different numbers for the same eight ranks.
"""
world = {"size": 8}
def _base_init(
self,
*,
tp_size: int = 8,
ep_size: int = 1,
hidden: int = HIDDEN_SIZE,
is_sequence_parallel: bool = False,
has_up_proj: bool = True,
has_shared_experts: bool = True,
routed_scaling_factor: float = 1.0,
) -> None:
self.moe_config = SimpleNamespace(
tp_size=tp_size,
ep_size=ep_size,
is_sequence_parallel=is_sequence_parallel,
)
self.routed_output_transform = SimpleNamespace(
norm=None,
up_proj=SimpleNamespace(weight=torch.zeros(hidden, LATENT_SIZE))
if has_up_proj
else None,
)
self.routed_scaling_factor = routed_scaling_factor
self._shared_experts = object() if has_shared_experts else None
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
monkeypatch.setattr(
latent_moe_runner,
"get_tensor_model_parallel_world_size",
lambda: world["size"],
)
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
world["size"] = tp_world
return ROCmLatentMoERunner(**kwargs)
return _build
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
build_runner,
) -> None:
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
but the hidden dim is still split across all eight TP ranks, so the tail
must shard exactly as it does without EP.
"""
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
build_runner,
) -> None:
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
reports 4 for a TP process group of 2.
Only those 2 ranks share a hidden dim and take part in the final
all-reduce, so sharding by 4 would leave half of it with no routed
contribution on either rank.
"""
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
assert runner._tail_shardable
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
@pytest.mark.parametrize(
"override",
[
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
pytest.param({"has_up_proj": False}, id="no-up-proj"),
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
pytest.param({"is_sequence_parallel": True}, id="sequence-parallel"),
pytest.param({"routed_scaling_factor": 2.0}, id="routed-scaling-factor"),
],
)
def test_falls_back_when_the_config_breaks_the_shard(
build_runner, override: dict
) -> None:
"""Each of these makes the sharded tail wrong, not merely slower.
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
so there is nothing to shard and no all-reduce to stitch shards back
together, however large ``moe_config.tp_size`` claims to be.
"""
runner = build_runner(**override)
assert not runner._tail_shardable
assert runner._up_proj_shard_size == 0
@@ -1,119 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm.models.kimi_k3.amd.linear import KimiDecoderLayer, KimiMLAAttention
class _DirectMLA(KimiMLAAttention):
def __init__(self, result: torch.Tensor) -> None:
nn.Module.__init__(self)
self.result = result
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
return self.result
def _make_layer(self_attn: nn.Module) -> KimiDecoderLayer:
layer = object.__new__(KimiDecoderLayer)
nn.Module.__init__(layer)
layer.self_attn = self_attn
return layer
def test_mla_self_attention_returns_projection_storage_directly() -> None:
hidden_states = torch.randn(4, 8)
projected = torch.randn_like(hidden_states)
layer = _make_layer(_DirectMLA(projected))
output = layer._run_self_attn(torch.arange(4), hidden_states)
assert output is projected
assert output.data_ptr() == projected.data_ptr()
@pytest.mark.parametrize("quantized", [False, True])
@pytest.mark.parametrize("fused_decode", [False, True])
def test_mla_forward_preserves_output_shape_with_fp8_input(
monkeypatch, quantized: bool, fused_decode: bool
) -> None:
"""Both MLA paths accept producer-quantized input after decode fusion."""
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
)
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
hidden_states = torch.randn(4, 8, dtype=torch.bfloat16)
if quantized:
hidden_states = QuantizedActivation(
hidden_states.to(kFp8DynamicTokenSym.dtype),
torch.ones(4, 1),
hidden_states.dtype,
hidden_states.shape,
kFp8DynamicTokenSym,
)
projected = torch.randn(4, 8, dtype=torch.bfloat16)
paths = []
def input_projection(value):
assert value is hidden_states
return torch.zeros(4, 8, dtype=torch.bfloat16), None
def attention_output(path, output_shape):
assert output_shape == projected.shape
paths.append(path)
return projected
attention = SimpleNamespace(
q_lora_rank=2,
kv_lora_rank=4,
qk_rope_head_dim=2,
qk_head_dim=4,
num_heads=2,
v_head_dim=4,
fused_qkv_a_proj=input_projection,
q_a_layernorm=nn.Identity(),
_normalize_q_kv=lambda q, kv: (q, kv),
q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None),
rotary_emb=None,
indexer=None,
dcp_q_replicate=False,
_fused_qk_prep=fused_decode,
_fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output(
"fused", shape
),
g_proj=None,
o_proj=lambda value: (value, None),
)
if fused_decode:
attention.mla_attn = SimpleNamespace(layer_name="test")
monkeypatch.setattr(
"vllm.models.kimi_k3.amd.mla.get_attention_context",
lambda _: (
SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4),
None,
torch.zeros(1),
torch.arange(4),
),
)
else:
attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output(
"regular", kwargs["output_shape"]
)
output = KimiK3MultiHeadLatentAttentionWrapper.forward(
attention, torch.arange(4), hidden_states
)
assert output is projected
assert paths == ["fused" if fused_decode else "regular"]
-302
View File
@@ -1,302 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
import pytest
import torch
import torch.nn.functional as F
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
from vllm.models.kimi_k3.nvidia.ops import attn_res
from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
MAX_BLOCKS = 8
EPS = 1e-5
attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res")
def _on_rocm_below_10() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import get_rocm_version
return (get_rocm_version() or (0,)) < (10,)
# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD
# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only
# the num_blocks > 0 loop contains it. ROCm serves AttnRes from
# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost.
_OLD_ROCM = _on_rocm_below_10()
def _skip_on_old_rocm(num_blocks: int) -> None:
if _OLD_ROCM and num_blocks > 0:
pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select")
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
storage = torch.randn(
*shape[:-1],
shape[-1] + padding,
device="cuda",
dtype=torch.bfloat16,
)
return storage[..., : shape[-1]]
def _reference(
prefix: torch.Tensor,
delta: torch.Tensor | None,
blocks: torch.Tensor,
norm_weight: torch.Tensor,
qk_weight: torch.Tensor,
output_norm_weight: torch.Tensor | None,
num_blocks: int,
) -> tuple[torch.Tensor, torch.Tensor]:
if delta is not None:
prefix = prefix + delta
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
keys = F.rms_norm(values, (HIDDEN_SIZE,), norm_weight, EPS)
probs = (keys @ qk_weight).softmax(dim=-1)
output = torch.matmul(probs.unsqueeze(1), values).squeeze(1)
if output_norm_weight is not None:
output = F.rms_norm(output, (HIDDEN_SIZE,), output_norm_weight, EPS)
return output, prefix
@pytest.mark.parametrize(
(
"num_tokens",
"num_blocks",
"row_padding",
"write_block",
"has_delta",
"backend",
),
[
pytest.param(1, 0, 0, True, False, "triton", id="triton-empty"),
pytest.param(1, 0, 0, True, True, "triton", id="triton-empty-add"),
pytest.param(17, 5, 7, True, False, "triton", id="triton-write"),
pytest.param(17, 5, 7, True, True, "triton", id="triton-write-add"),
pytest.param(3, 8, 0, False, False, "triton", id="triton-full"),
pytest.param(3, 8, 0, False, True, "triton", id="triton-full-add"),
pytest.param(1, 4, 0, False, True, "nvidia", id="nvidia-single-token"),
pytest.param(320, 1, 0, False, True, "nvidia", id="nvidia-1"),
pytest.param(320, 4, 0, False, True, "nvidia", id="nvidia-4"),
pytest.param(320, 8, 0, False, True, "nvidia", id="nvidia-8"),
],
)
def test_attn_res(
num_tokens: int,
num_blocks: int,
row_padding: int,
write_block: bool,
has_delta: bool,
backend: str,
):
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
pytest.skip("NVIDIA AttnRes requires the SM100 family")
_skip_on_old_rocm(num_blocks)
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
delta = (
_randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
if has_delta
else None
)
blocks = _randn_with_row_padding(
num_tokens, MAX_BLOCKS, HIDDEN_SIZE, padding=row_padding
)
norm_weight = 1 + 0.1 * torch.randn(
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
output_norm_weight = 1 + 0.1 * torch.randn(
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
original_blocks = blocks.clone()
expected, expected_prefix = _reference(
prefix.clone(),
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
)
block_write_idx = num_blocks if write_block else -1
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
block_write_idx,
EPS,
EPS,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
if write_block:
original_blocks[:, block_write_idx].copy_(expected_prefix)
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
def test_attn_res_block_counts(num_blocks: int):
_skip_on_old_rocm(num_blocks)
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
blocks = torch.randn(
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
output_norm_weight = torch.ones_like(norm_weight)
expected, _ = _reference(
prefix.clone(),
None,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
)
actual = attn_res(
prefix,
None,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
-1,
EPS,
EPS,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
def test_attn_res_without_output_norm():
_skip_on_old_rocm(MAX_BLOCKS)
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
delta = torch.randn_like(prefix)
blocks = torch.randn(
7, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
norm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
qk_weight = (
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
)
expected, _ = _reference(
prefix.clone(), delta, blocks, norm_weight, qk_weight, None, MAX_BLOCKS
)
actual = attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
None,
MAX_BLOCKS,
-1,
EPS,
0.0,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
@pytest.mark.parametrize(
("num_blocks", "has_delta", "has_output_norm", "write_block"),
[
(0, False, True, True),
(4, True, True, True),
(4, False, True, False),
(8, True, False, False),
],
)
def test_sm100_variants_do_not_fall_back_to_triton(
monkeypatch,
num_blocks: int,
has_delta: bool,
has_output_norm: bool,
write_block: bool,
):
if not current_platform.is_device_capability_family(100) or not hasattr(
torch.ops._C, "kimi_k3_attn_res"
):
pytest.skip("native AttnRes requires an SM100 build")
class FailingKernel:
def __getitem__(self, grid):
raise AssertionError(f"unexpected Triton launch with grid {grid}")
monkeypatch.setattr(attn_res_module, "_attn_res_kernel", FailingKernel())
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
delta = torch.zeros_like(prefix) if has_delta else None
blocks = torch.randn(
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
)
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
qk_weight = torch.randn_like(norm_weight) / HIDDEN_SIZE**0.5
output_norm_weight = torch.ones_like(norm_weight) if has_output_norm else None
attn_res(
prefix,
delta,
blocks,
norm_weight,
qk_weight,
output_norm_weight,
num_blocks,
num_blocks if write_block else -1,
EPS,
EPS,
)
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
def test_fused_mtp_input(num_tokens: int):
positions = torch.arange(num_tokens, device="cuda")
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
previous_hidden_states = _randn_with_row_padding(
num_tokens, HIDDEN_SIZE, padding=11
)
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
expected = torch.cat(
(
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
),
dim=-1,
)
actual = fused_mtp_input(
positions,
inputs_embeds,
previous_hidden_states,
enorm_weight,
hnorm_weight,
EPS,
)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
assert actual.is_contiguous()
@@ -1,195 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Which value the DFlash drafter is fed under AttnRes.
`_capture_aux_hidden_stream` picks the weights it mixes against from one of
three places depending on where the tapped layer sits, and returns the plain
running prefix when the feature is off. The mixture itself is the kernel's
job and is covered by ``test_attn_res.py``; what is asserted here is the
selection, which is the part that can silently feed the drafter the wrong
tensor.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.models.kimi_k3.nvidia import model as k3_model
END_LAYER = 4
def _weights(tag: float) -> SimpleNamespace:
"""A norm/projection pair that is identifiable by value."""
return SimpleNamespace(
weight=torch.full((2,), tag),
variance_epsilon=tag,
)
def _stub_model(*, enabled: bool, use_attn_res: bool = True) -> SimpleNamespace:
"""A stand-in carrying only what the tap reads.
Constructing the real model needs a distributed init and weights, and none
of it participates in the selection under test.
"""
consumers = []
for i in range(END_LAYER):
consumers.append(
SimpleNamespace(
self_attention_res_norm=_weights(float(i)),
self_attention_res_proj=SimpleNamespace(
weight=torch.full((1, 2), float(i))
),
prev_valid_blocks=i,
)
)
return SimpleNamespace(
_aux_attn_res_stream=enabled,
use_attn_res=use_attn_res,
end_layer=END_LAYER,
layers=consumers,
output_attn_res_norm=_weights(99.0),
output_attn_res_proj=SimpleNamespace(weight=torch.full((1, 2), 99.0)),
num_attn_res_blocks=99,
)
@pytest.fixture
def recorder(monkeypatch):
"""Replace the kernel so the call it would have made is inspectable."""
calls = []
def _fake_attn_res(
prefix,
delta,
block_residual,
norm_weight,
proj_weight,
output_norm_weight,
**kwargs,
):
calls.append(
SimpleNamespace(
prefix=prefix,
delta=delta,
block_residual=block_residual,
norm_weight=norm_weight,
proj_weight=proj_weight,
kwargs=kwargs,
)
)
return torch.full_like(prefix, -1.0)
monkeypatch.setattr(k3_model, "attn_res", _fake_attn_res)
return calls
def _set_last_rank(monkeypatch, is_last: bool):
monkeypatch.setattr(
k3_model,
"get_pp_group",
lambda: SimpleNamespace(is_last_rank=is_last),
)
def _call(stub, layer_idx, prefix_sum, pending_mlp_out, block_residual):
return k3_model.KimiLinearModel._capture_aux_hidden_stream(
stub, layer_idx, prefix_sum, pending_mlp_out, block_residual
)
@pytest.mark.parametrize(
"enabled,use_attn_res", [(False, True), (True, False), (False, False)]
)
def test_disabled_reproduces_the_plain_residual_sum(
recorder, monkeypatch, enabled, use_attn_res
):
"""Off, the tap must be exactly the sum it replaced.
Both conditions matter. `use_attn_res` is what constructs the norm and
projection weights, so without it the lookups below would raise rather
than fall back.
"""
_set_last_rank(monkeypatch, True)
prefix_sum = torch.tensor([1.0, 2.0])
pending = torch.tensor([0.5, 0.25])
got = _call(
_stub_model(enabled=enabled, use_attn_res=use_attn_res),
0,
prefix_sum,
pending,
torch.zeros(2),
)
torch.testing.assert_close(got, prefix_sum + pending)
assert not recorder, "the kernel must not run when the tap is off"
def test_taps_the_consumer_layer_when_one_follows(recorder, monkeypatch):
"""The value the next layer reads is the mixture against *its* weights,
so the tap has to reach forward rather than use the current layer's."""
_set_last_rank(monkeypatch, True)
_call(_stub_model(enabled=True), 1, torch.zeros(2), None, torch.zeros(2))
assert len(recorder) == 1
call = recorder[0]
# Layer 2's weights, not layer 1's.
torch.testing.assert_close(call.norm_weight, torch.full((2,), 2.0))
assert call.kwargs["num_blocks"] == 2
def test_last_layer_on_the_final_rank_uses_the_output_aggregation(
recorder, monkeypatch
):
"""Nothing downstream but the model's own output-side mixture."""
_set_last_rank(monkeypatch, True)
_call(
_stub_model(enabled=True), END_LAYER - 1, torch.zeros(2), None, torch.zeros(2)
)
assert len(recorder) == 1
torch.testing.assert_close(recorder[0].norm_weight, torch.full((2,), 99.0))
assert recorder[0].kwargs["num_blocks"] == 99
def test_aux_layer_at_non_final_pp_boundary_is_rejected(monkeypatch):
model = k3_model.KimiLinearModel.__new__(k3_model.KimiLinearModel)
torch.nn.Module.__init__(model)
model.use_attn_res = True
model.end_layer = 72
monkeypatch.setattr(
"vllm.distributed.parallel_state.model_parallel_is_initialized", lambda: False
)
_set_last_rank(monkeypatch, False)
monkeypatch.setattr(
k3_model.KimiLinearModel,
"_aux_attn_res_stream",
property(lambda self: True),
)
with pytest.raises(ValueError, match="Auxiliary layer 72"):
model._set_aux_hidden_state_layers((3, 24, 48, 72, 90))
def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
recorder, monkeypatch
):
"""The kernel writes an applied delta back into the prefix in place, which
would double-add it into the live residual stream, so the pending output
has to arrive already summed into the prefix with `delta` left None."""
_set_last_rank(monkeypatch, True)
prefix_sum = torch.tensor([1.0, 2.0])
pending = torch.tensor([0.5, 0.25])
_call(_stub_model(enabled=True), 0, prefix_sum, pending, torch.zeros(2))
assert len(recorder) == 1
assert recorder[0].delta is None
torch.testing.assert_close(recorder[0].prefix, prefix_sum + pending)
# And the caller's tensor is not mutated on the way.
torch.testing.assert_close(prefix_sum, torch.tensor([1.0, 2.0]))
-202
View File
@@ -1,202 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import torch
from vllm.model_executor.models.interfaces import supports_eagle3
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia.model import (
KimiK3ForConditionalGeneration,
KimiLinearForCausalLM,
KimiLinearModel,
)
def _make_kimi_linear_model() -> KimiLinearModel:
model = object.__new__(KimiLinearModel)
object.__setattr__(model, "aux_hidden_state_layers", (2,))
object.__setattr__(model, "use_sequence_parallel", False)
object.__setattr__(model, "use_attn_res", False)
return model
def test_kimi_k3_advertises_eagle3_support():
assert supports_eagle3(KimiK3ForConditionalGeneration)
def test_kimi_linear_advertises_eagle3_support():
# The text-only architecture serves the same inner KimiLinearModel, which
# already carries the EagleModelMixin tap machinery - only the interface
# declaration was missing, so EAGLE3-family speculative decoding (dspark)
# was rejected at startup with "Model does not support EAGLE3 interface".
assert supports_eagle3(KimiLinearForCausalLM)
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
target = object.__new__(KimiK3ForConditionalGeneration)
torch.nn.Module.__init__(target)
model = _make_kimi_linear_model()
object.__setattr__(model, "layers", [None] * 93)
language_model = SimpleNamespace(
embed_input_ids=lambda _: None,
forward=lambda input_ids, positions: None,
model=model,
)
object.__setattr__(target, "language_model", language_model)
object.__setattr__(target, "_language_model_names", ["language_model"])
target.set_aux_hidden_state_layers((2, 46, 90))
assert model.aux_hidden_state_layers == (2, 46, 90)
assert target.get_eagle3_default_aux_hidden_state_layers() == (
2,
46,
90,
)
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
layer_residual = torch.tensor([[5.0, 6.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", False)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
expected_layer_output = layer_hidden_states + layer_residual
torch.testing.assert_close(output, expected_layer_output)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
prefix_sum = torch.tensor([[5.0, 6.0]])
block_residual = torch.tensor([[[7.0, 8.0]]])
final_hidden_states = torch.tensor([[9.0, 10.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
object.__setattr__(model, "use_attn_res", True)
object.__setattr__(model, "num_attn_res_blocks", 1)
object.__setattr__(
model,
"output_attn_res_norm",
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
)
object.__setattr__(
model,
"output_attn_res_proj",
SimpleNamespace(weight=torch.ones(1, 2)),
)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
final_attn_res = Mock(return_value=final_hidden_states)
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
output, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
torch.testing.assert_close(output, final_hidden_states)
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
assert final_attn_res.call_args.args[2] is block_residual
def test_attn_res_stream_capture_receives_the_layer_outputs_in_order(monkeypatch):
"""Pin the argument mapping at the call site.
The capture helper's own tests invoke it directly by keyword, so they
cannot catch a swap where `forward` hands it the residual as the pending
MLP output. Both are tensors of the same shape, so a swap is silent: it
feeds the drafter a wrong but well-formed tensor.
"""
model = _make_kimi_linear_model()
initial_hidden_states = torch.tensor([[1.0, 2.0]])
layer_hidden_states = torch.tensor([[3.0, 4.0]])
prefix_sum = torch.tensor([[5.0, 6.0]])
block_residual = torch.tensor([[[7.0, 8.0]]])
captured = torch.tensor([[11.0, 12.0]])
object.__setattr__(model, "start_layer", 0)
object.__setattr__(model, "end_layer", 1)
object.__setattr__(
model,
"layers",
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
)
object.__setattr__(model, "aux_hidden_state_layers", (1,))
object.__setattr__(model, "use_attn_res", True)
object.__setattr__(model, "num_attn_res_blocks", 1)
object.__setattr__(
model,
"output_attn_res_norm",
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
)
object.__setattr__(
model,
"output_attn_res_proj",
SimpleNamespace(weight=torch.ones(1, 2)),
)
monkeypatch.setattr(
kimi_model,
"get_pp_group",
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
)
monkeypatch.setattr(kimi_model, "attn_res", Mock(return_value=torch.zeros(1, 2)))
monkeypatch.setenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "1")
capture = Mock(return_value=captured)
monkeypatch.setattr(KimiLinearModel, "_capture_aux_hidden_stream", capture)
_, aux_hidden_states = model.forward(
input_ids=None,
positions=torch.tensor([0]),
intermediate_tensors=None,
inputs_embeds=initial_hidden_states,
)
layer_idx, got_prefix, got_pending, got_residual = capture.call_args.args
assert layer_idx == 0
assert got_prefix is prefix_sum
assert got_pending is layer_hidden_states
assert got_residual is block_residual
torch.testing.assert_close(aux_hidden_states[0], captured)
-104
View File
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from collections.abc import Callable
from types import ModuleType, SimpleNamespace
from unittest.mock import patch
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.mxfp4 import (
_moe_weight_override_is_int4,
_use_k3_situ_int4_gfx942,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
pytestmark = pytest.mark.cpu_test
class _FakeRocmModule(ModuleType):
on_gfx942: Callable[[], bool]
class _FakeAiterOpsModule(ModuleType):
rocm_aiter_ops: SimpleNamespace
def _vllm_config(moe_weight: str | None):
quantization_config = (
None
if moe_weight is None
else QuantizationConfigArgs(moe={"weight": moe_weight})
)
return SimpleNamespace(
model_config=SimpleNamespace(quantization_config=quantization_config)
)
@pytest.mark.parametrize(
("moe_weight", "expected"),
[
(None, False),
("mxfp4", False),
("int4_per_group_32", True),
],
)
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
monkeypatch.setattr(
"vllm.config.get_current_vllm_config",
lambda: _vllm_config(moe_weight),
)
assert _moe_weight_override_is_int4() is expected
@pytest.mark.parametrize("int4_requested", [False, True])
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
moe = SimpleNamespace(
activation=MoEActivation.SITU,
activation_situ_linear_beta=25.0,
)
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
fake_rocm.on_gfx942 = lambda: True
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
with (
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
patch.dict(
sys.modules,
{
"vllm.platforms.rocm": fake_rocm,
"vllm._aiter_ops": fake_aiter_ops,
},
),
patch(
"vllm.model_executor.layers.quantization.mxfp4."
"_moe_weight_override_is_int4",
return_value=int4_requested,
),
):
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
def test_int4_per_group_32_is_not_an_online_moe_method():
# Load-time gfx942 requant. Must not match the online MoE table.
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
)
assert (
config.resolve_quant_method_cls(RoutedExperts, "model.layers.0.mlp.experts")
is None
)
def test_unknown_online_moe_weight_still_raises():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
)
with pytest.raises(ValueError, match="online quantization"):
config.resolve_quant_method_cls(RoutedExperts, "model.layers.0.mlp.experts")
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,482 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import ray
import torch
import torch.distributed as dist
import torch.nn.functional as F
from tests.utils import (
init_test_distributed_environment,
multi_gpu_test,
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
from vllm.models.kimi_k3.nvidia import latent_moe_runner
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
from vllm.platforms import current_platform
HIDDEN_SIZE = 7168
LATENT_SIZE = 3584
EPS = 0.1
TOP_K = 8
def test_deferred_finalize_enabled_before_moe_kernel_setup(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class FakeMoEConfig:
tp_size = 8
dp_size = 1
ep_size = 1
pcp_size = 1
is_sequence_parallel = False
hidden_dim = LATENT_SIZE
hidden_dim_unpadded = LATENT_SIZE
experts_per_token = 16
_defer_moe_finalize = False
defer_moe_finalize_max_num_tokens = -1
defer_moe_finalize = FusedMoEConfig.defer_moe_finalize
limit_deferred_moe_finalize = FusedMoEConfig.limit_deferred_moe_finalize
use_deferred_moe_finalize = FusedMoEConfig.use_deferred_moe_finalize
moe_config = FakeMoEConfig()
quant_method = SimpleNamespace(
experts_cls=TrtLlmMxfp4ExpertsMonolithic,
moe_kernel=None,
)
norm_weight = torch.empty(LATENT_SIZE, dtype=torch.bfloat16)
transform = SimpleNamespace(
norm=SimpleNamespace(weight=norm_weight, variance_epsilon=EPS),
up_proj=SimpleNamespace(
weight=SimpleNamespace(shape=(HIDDEN_SIZE, LATENT_SIZE))
),
)
def fake_runner_init(runner, *args, **kwargs) -> None:
runner.moe_config = moe_config
runner.routed_experts = SimpleNamespace(quant_method=quant_method)
runner._shared_experts = object()
runner.routed_output_transform = transform
initialized_with: dict[str, object] = {}
tail_op = SimpleNamespace(contract=SimpleNamespace(max_num_tokens=128))
def fake_tail_initialize(**kwargs):
initialized_with.update(kwargs)
return tail_op
monkeypatch.setattr(MoERunner, "__init__", fake_runner_init)
monkeypatch.setattr(latent_moe_runner.torch.cuda, "Event", lambda: object())
monkeypatch.setattr(
latent_moe_runner,
"current_platform",
SimpleNamespace(
is_cuda=lambda: True,
is_device_capability_family=lambda capability: capability == 100,
current_device=lambda: torch.device("cuda"),
),
)
monkeypatch.setattr(
latent_moe_runner,
"get_current_vllm_config",
lambda: SimpleNamespace(
parallel_config=SimpleNamespace(use_ubatching=False),
model_config=SimpleNamespace(enable_sleep_mode=False),
),
)
monkeypatch.setattr(KimiK3LatentMoETailOp, "initialize", fake_tail_initialize)
latent_moe_runner.LatentMoERunner()
assert moe_config.use_deferred_moe_finalize
assert moe_config.defer_moe_finalize_max_num_tokens == 128
assert initialized_with["experts_per_token"] == 16
def _make_deferred_routed_output(
num_tokens: int,
device: torch.device,
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
num_routes = num_tokens * TOP_K
num_permuted_rows = num_routes + 7
expanded_output = torch.randn(
num_routes,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
expert_weights = torch.rand(
num_tokens,
TOP_K,
device=device,
dtype=torch.bfloat16,
)
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
gemm2_permuted = torch.empty(
num_permuted_rows,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
gemm2_permuted[expanded_idx] = expanded_output
finalized = torch.zeros(
num_tokens,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
expanded_output = expanded_output.view(num_tokens, TOP_K, LATENT_SIZE)
for slot in range(TOP_K):
finalized.add_(expanded_output[:, slot] * expert_weights[:, slot, None])
return (
UnfinalizedMoEOutput(
gemm2_permuted=gemm2_permuted,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32).view(
num_tokens, TOP_K
),
),
finalized,
)
def _make_bf16_top16_deferred_output(
num_tokens: int,
device: torch.device,
*,
drop_last_route: bool = False,
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
top_k = 16
num_routes = num_tokens * top_k
num_permuted_rows = num_routes + 7
expanded_output = torch.randn(
num_routes,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
expert_weights = torch.rand(
num_tokens,
top_k,
device=device,
dtype=torch.bfloat16,
)
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
gemm2_permuted = torch.empty(
num_permuted_rows,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
gemm2_permuted[expanded_idx] = expanded_output
expanded_output = expanded_output.view(num_tokens, top_k, LATENT_SIZE)
expanded_idx = expanded_idx.view(num_tokens, top_k)
if drop_last_route:
expanded_output[:, -1].zero_()
expanded_idx[:, -1] = -1
finalized = (
(expanded_output.float() * expert_weights[:, :, None].float())
.sum(1)
.to(torch.bfloat16)
)
return (
UnfinalizedMoEOutput(
gemm2_permuted=gemm2_permuted,
expert_weights=expert_weights,
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32),
),
finalized,
)
@ray.remote(num_gpus=1, max_calls=1)
def _test_latent_moe_tail_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
)
torch.manual_seed(0)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
up_weight = (
torch.randn(
HIDDEN_SIZE,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
/ LATENT_SIZE**0.5
)
group = get_tp_group().device_group
op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
)
cutedsl_warmup()
# M=65 crosses two 32-CTA token waves and reuses the first DSM slot.
for iteration, num_tokens in enumerate((1, 5, 8, 16, 33, 65, 5)):
torch.manual_seed(100 * iteration + rank + 1)
routed_output = torch.randn(
num_tokens,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
).mul_(0.01)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
routed_reference = routed_output.clone()
shared_reference = shared_output.clone()
dist.all_reduce(routed_reference, group=group)
dist.all_reduce(shared_reference, group=group)
expected = F.linear(
F.rms_norm(
routed_reference,
(LATENT_SIZE,),
rms_weight,
EPS,
),
up_weight,
)
expected.add_(shared_reference)
actual = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
assert actual.is_contiguous()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_output = op(
routed_output,
shared_output,
rms_weight,
up_weight,
)
graph.replay()
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
def _run_latent_moe_tail_test(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
) -> None:
if not current_platform.is_device_capability_family(100):
pytest.skip("K3 latent-MoE tail fusion requires SM100")
multi_process_parallel(
monkeypatch,
tp_size,
1,
_test_latent_moe_tail_worker,
)
@ray.remote(num_gpus=1, max_calls=1)
def _test_deferred_finalize_parity_worker(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
pp_size: int,
rank: int,
distributed_init_port: str,
) -> None:
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
tp_size,
pp_size,
rank,
distributed_init_port,
)
torch.manual_seed(1000 + rank)
rms_weight = 1 + 0.1 * torch.randn(
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
up_weight = (
torch.randn(
HIDDEN_SIZE,
LATENT_SIZE,
device=device,
dtype=torch.bfloat16,
)
/ LATENT_SIZE**0.5
)
finalized_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
)
deferred_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
experts_per_token=TOP_K,
)
bf16_deferred_op = KimiK3LatentMoETailOp.initialize(
hidden_size=HIDDEN_SIZE,
latent_size=LATENT_SIZE,
dtype=torch.bfloat16,
device=device,
rms_eps=EPS,
experts_per_token=16,
)
cutedsl_warmup()
for iteration, num_tokens in enumerate((1, 5, 16, 33, 65)):
torch.manual_seed(2000 + 100 * iteration + rank)
deferred_output, finalized_output = _make_deferred_routed_output(
num_tokens,
device,
)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
expected = finalized_op(
finalized_output,
shared_output,
rms_weight,
up_weight,
)
actual = deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
for iteration, num_tokens in enumerate((1, 2, 4, 5, 16)):
torch.manual_seed(3000 + 100 * iteration + rank)
deferred_output, finalized_output = _make_bf16_top16_deferred_output(
num_tokens,
device,
drop_last_route=num_tokens == 5,
)
shared_output = torch.randn(
num_tokens,
HIDDEN_SIZE,
device=device,
dtype=torch.bfloat16,
)
expected = finalized_op(
finalized_output,
shared_output,
rms_weight,
up_weight,
)
actual = bf16_deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
if num_tokens == 4:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_actual = bf16_deferred_op(
deferred_output,
shared_output,
rms_weight,
up_weight,
)
graph.replay()
torch.testing.assert_close(
graph_actual,
expected,
atol=8e-2,
rtol=3e-2,
)
def _run_deferred_finalize_parity_test(
monkeypatch: pytest.MonkeyPatch,
tp_size: int,
) -> None:
if not current_platform.is_device_capability_family(100):
pytest.skip("K3 latent-MoE tail fusion requires SM100")
multi_process_parallel(
monkeypatch,
tp_size,
1,
_test_deferred_finalize_parity_worker,
)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_tp8_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 8)
@multi_gpu_test(num_gpus=16)
def test_latent_moe_tail_tp16_matches_native_path(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_latent_moe_tail_test(monkeypatch, 16)
@multi_gpu_test(num_gpus=8)
def test_latent_moe_tail_deferred_finalize_matches_finalized(
monkeypatch: pytest.MonkeyPatch,
) -> None:
_run_deferred_finalize_parity_test(monkeypatch, 8)
@@ -1,433 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""K3's fused chunked-context prefill must match the generic MLA impl.
The layer owns its own context loop so it can fuse the per-chunk K/V pack and
skip re-quantizing an already-quantized query. That is only safe if it feeds the
prefill backend exactly what ``MLACommonBaseImpl._compute_prefill_context``
would, chunk for chunk.
"""
from types import SimpleNamespace
import pytest
import torch
import vllm.model_executor.layers.attention.mla_attention as mla_attention
from vllm.model_executor.layers.attention.mla_attention import (
MLACommonBaseImpl,
MLACommonImpl,
MLACommonPrefillMetadata,
build_mla_chunked_context_metadata,
)
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
)
_KV_LORA_RANK = 512
_QK_NOPE = 128
_QK_ROPE = 64
_V_HEAD_DIM = 128
_ENTRY = _KV_LORA_RANK + _QK_ROPE
_NUM_HEADS = 2
_BLOCK_SIZE = 16
_WORKSPACE_TOKENS = 128
# Splits one long request across chunks, packs short ones together, and leaves
# the last request without any context.
_CONTEXT_LENS = [200, 48, 32, 0]
_QUERY_LENS = [8, 4, 6, 5]
class _RecordingPrefillBackend:
"""Records what each chunk is asked to attend over.
``honors_out`` mimics the two backend families: one that writes into a
caller-provided ``out`` (trtllm_ragged, flashinfer, tokenspeed) and one that
always returns its own buffer (flash_attn with a padded V, aiter).
"""
def __init__(self, honors_out: bool = False) -> None:
self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
self.out_destinations: list[torch.Tensor | None] = []
self._honors_out = honors_out
@staticmethod
def get_name() -> str:
return "recording"
def supports_out(self) -> bool:
return self._honors_out
def run_prefill_context_chunk(self, *, chunk, q, k, v, out=None):
self.calls.append((q.float().clone(), k.float().clone(), v.float().clone()))
self.out_destinations.append(out)
assert out is None or self._honors_out
# Fold K/V into the partial so any packing difference shows up in the
# merged context output, not just in the recorded calls.
digest = (k.float().mean() + v.float().mean()).item()
num_q = q.shape[0]
if out is None:
out = torch.empty(
(num_q, _NUM_HEADS, _V_HEAD_DIM), device=q.device, dtype=torch.bfloat16
)
else:
assert out.shape == (num_q, _NUM_HEADS, _V_HEAD_DIM)
out.fill_(digest)
lse = torch.full(
(_NUM_HEADS, num_q),
1.0 + chunk.index,
device=q.device,
dtype=torch.float32,
)
return out, lse
class _KVBProj(torch.nn.Module):
"""Stand-in for the layer's ``kv_b_proj`` (returns an (out, bias) tuple).
Enforces the same input contract as the real linear methods, because that is
what decides whether the gathered latent needs a cast:
* an fp8 weight consumes the fp8 latent directly and dequantizes internally,
so it takes fp8 or bf16;
* a bf16 weight -- what a stock K3 checkpoint carries -- is a plain
``F.linear`` and rejects anything but bf16, exactly as torch does.
Either weight dtype produces a bf16 output.
"""
def __init__(self, device: torch.device, weight_dtype: torch.dtype) -> None:
super().__init__()
weight = (
torch.randn(
_NUM_HEADS * (_QK_NOPE + _V_HEAD_DIM),
_KV_LORA_RANK,
device=device,
dtype=torch.bfloat16,
)
* 0.05
)
self.register_buffer("weight", weight.to(weight_dtype))
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
if self.weight.dtype == torch.bfloat16 and x.dtype != torch.bfloat16:
raise RuntimeError(
"a bfloat16 kv_b_proj cannot consume the gathered latent as "
f"{x.dtype}; it must be cast first"
)
return torch.nn.functional.linear(
x.to(torch.bfloat16), self.weight.to(torch.bfloat16)
), None
class _FusedLayer:
"""Only the attributes K3's context loop reads."""
_compute_prefill_context = MultiHeadLatentAttention._compute_prefill_context
_gather_context_latent = MultiHeadLatentAttention._gather_context_latent
_attn_read_kv_cache = MultiHeadLatentAttention._attn_read_kv_cache
def __init__(self, kv_b_proj, kv_cache, kv_cache_dtype, k_scale) -> None:
self.kv_b_proj = kv_b_proj
self.kv_cache = kv_cache
self.kv_cache_dtype = kv_cache_dtype
self._k_scale = k_scale
self.kv_lora_rank = _KV_LORA_RANK
self.num_local_heads = _NUM_HEADS
self.qk_nope_head_dim = _QK_NOPE
self.v_head_dim = _V_HEAD_DIM
class _ReferenceImpl:
"""Only the attributes the generic context loop reads."""
_compute_prefill_context = MLACommonBaseImpl._compute_prefill_context
_concat_k_nope_k_pe = MLACommonBaseImpl._concat_k_nope_k_pe
_use_flashinfer_concat_mla_k = False
def __init__(self, kv_b_proj, kv_cache_dtype) -> None:
self.kv_b_proj = kv_b_proj
self.kv_cache_dtype = kv_cache_dtype
self.kv_lora_rank = _KV_LORA_RANK
self.num_heads = _NUM_HEADS
self.qk_nope_head_dim = _QK_NOPE
self.qk_rope_head_dim = _QK_ROPE
self.v_head_dim = _V_HEAD_DIM
def _build_prefill_metadata(
device: torch.device,
workspace_dtype: torch.dtype,
q_data_type: torch.dtype,
backend: _RecordingPrefillBackend,
) -> MLACommonPrefillMetadata:
query_start_loc_cpu = torch.zeros(len(_QUERY_LENS) + 1, dtype=torch.int32)
query_start_loc_cpu[1:] = torch.tensor(_QUERY_LENS, dtype=torch.int32).cumsum(0)
workspace = torch.empty(
(_WORKSPACE_TOKENS, _ENTRY), dtype=workspace_dtype, device=device
)
chunked_context = build_mla_chunked_context_metadata(
context_lens_cpu=torch.tensor(_CONTEXT_LENS, dtype=torch.int32),
prefill_query_start_loc_cpu=query_start_loc_cpu,
chunked_prefill_workspace=workspace,
chunked_prefill_workspace_size=_WORKSPACE_TOKENS,
block_size=_BLOCK_SIZE,
align_chunk_to_block=True,
device=device,
dcp_world_size=1,
dcp_local_block_size=1,
dcp_virtual_block_size=1,
)
assert chunked_context is not None
assert len(chunked_context.chunks) > 1, "the batch must exercise accumulation"
max_blocks = (max(_CONTEXT_LENS) + max(_QUERY_LENS)) // _BLOCK_SIZE + 1
num_blocks = max_blocks * len(_CONTEXT_LENS)
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(
len(_CONTEXT_LENS), max_blocks
)
return MLACommonPrefillMetadata(
block_table=block_table,
query_start_loc=query_start_loc_cpu.to(device),
max_query_len=max(_QUERY_LENS),
chunked_context=chunked_context,
q_data_type=q_data_type,
output_dtype=torch.bfloat16,
prefill_backend=backend,
), num_blocks
@pytest.mark.parametrize("honors_out", [False, True], ids=["copy_out", "writes_out"])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize(
"kv_b_proj_quantized", [True, False], ids=["fp8_kv_b_proj", "bf16_kv_b_proj"]
)
@torch.inference_mode()
def test_fused_context_matches_generic_impl(
kv_b_proj_quantized: bool, kv_cache_dtype: str, honors_out: bool
) -> None:
"""Cache dtype and ``kv_b_proj`` dtype vary independently.
A stock K3 checkpoint pairs a bf16 ``kv_b_proj`` with an fp8 cache, so the
fused loop cannot assume the gathered latent is already in the dtype
``kv_b_proj`` accepts.
"""
torch.manual_seed(0)
device = torch.device("cuda")
fp8 = current_platform.fp8_dtype()
quantized = kv_cache_dtype == "fp8"
# An fp8 cache is read by an fp8 query, so the workspace keeps the fp8
# layout; a bf16 cache dequantizes into a bf16 workspace.
q_data_type = fp8 if quantized else torch.bfloat16
workspace_dtype = q_data_type
kv_b_proj = _KVBProj(
device, weight_dtype=fp8 if kv_b_proj_quantized else torch.bfloat16
)
k_scale = torch.ones(1, dtype=torch.float32, device=device)
backend_fused = _RecordingPrefillBackend(honors_out=honors_out)
# The reference impl never passes `out`, so it always allocates its own.
backend_ref = _RecordingPrefillBackend()
prefill_fused, num_blocks = _build_prefill_metadata(
device, workspace_dtype, q_data_type, backend_fused
)
prefill_ref, _ = _build_prefill_metadata(
device, workspace_dtype, q_data_type, backend_ref
)
cache = torch.randn(
(num_blocks, _BLOCK_SIZE, _ENTRY), device=device, dtype=torch.bfloat16
)
kv_cache = cache.to(fp8) if quantized else cache
q = (
torch.randn(
(sum(_QUERY_LENS), _NUM_HEADS, _QK_NOPE + _QK_ROPE),
device=device,
dtype=torch.bfloat16,
)
* 0.2
).to(q_data_type)
layer = _FusedLayer(kv_b_proj, kv_cache, kv_cache_dtype, k_scale)
fused_out, fused_lse = layer._compute_prefill_context(
q, SimpleNamespace(prefill=prefill_fused)
)
impl = _ReferenceImpl(kv_b_proj, kv_cache_dtype)
ref_out, ref_lse = impl._compute_prefill_context(
q, kv_cache, SimpleNamespace(prefill=prefill_ref), k_scale
)
assert len(backend_fused.calls) == len(backend_ref.calls)
for chunk_idx, (fused_call, ref_call) in enumerate(
zip(backend_fused.calls, backend_ref.calls, strict=True)
):
for name, fused_t, ref_t in zip(
("q", "k", "v"), fused_call, ref_call, strict=True
):
torch.testing.assert_close(
fused_t,
ref_t,
atol=0,
rtol=0,
msg=lambda m, n=name, i=chunk_idx: f"chunk {i} {n} differs: {m}",
)
torch.testing.assert_close(fused_out, ref_out, atol=0, rtol=0)
torch.testing.assert_close(fused_lse, ref_lse, atol=0, rtol=0)
# Every chunk but the continuation should have been written in place, i.e.
# straight into the returned accumulator, with no intermediate copy.
wrote_in_place = [
out is not None and out.data_ptr() == fused_out[chunk.token_slice].data_ptr()
for out, chunk in zip(
backend_fused.out_destinations,
prefill_fused.chunked_context.chunks,
strict=True,
)
]
continuations = [c.is_continuation for c in prefill_fused.chunked_context.chunks]
assert any(continuations), "the batch must exercise a continuation chunk"
if honors_out:
assert wrote_in_place == [not c for c in continuations]
else:
assert not any(wrote_in_place)
@torch.inference_mode()
def test_fused_context_rejects_an_unquantized_query() -> None:
"""The fp8 query is produced by the new-token epilogue, not re-cast here."""
torch.manual_seed(0)
device = torch.device("cuda")
fp8 = current_platform.fp8_dtype()
prefill, num_blocks = _build_prefill_metadata(
device, fp8, fp8, _RecordingPrefillBackend()
)
layer = _FusedLayer(
_KVBProj(device, weight_dtype=fp8),
torch.zeros((num_blocks, _BLOCK_SIZE, _ENTRY), device=device, dtype=fp8),
"fp8",
torch.ones(1, dtype=torch.float32, device=device),
)
q = torch.zeros(
(sum(_QUERY_LENS), _NUM_HEADS, _QK_NOPE + _QK_ROPE),
device=device,
dtype=torch.bfloat16,
)
with pytest.raises(AssertionError, match="new-token epilogue"):
layer._compute_prefill_context(q, SimpleNamespace(prefill=prefill))
# --------------------------------------------------------------------------
# Decode context parallelism: the generic impl keeps its loop (allgather +
# reorg) and only the per-chunk K/V pack is handed to the layer's fused kernel.
# --------------------------------------------------------------------------
_DCP = 2
_DCP_TOKENS = 48
def _run_dcp_context_loop(monkeypatch, *, q_data_type, fused_mla_kv_concat_fn):
"""Drive ``MLACommonImpl._context_parallel_compute_prefill_context`` on one
chunk with the collective pieces (gather, allgather, reorg) stubbed out."""
torch.manual_seed(0)
device = torch.device("cuda")
kv_b_proj = _KVBProj(device, weight_dtype=torch.bfloat16)
workspace = torch.randn(
(_DCP_TOKENS * (_DCP + 1), _ENTRY), dtype=torch.bfloat16, device=device
)
impl = SimpleNamespace(
kv_cache_dtype="auto",
kv_lora_rank=_KV_LORA_RANK,
qk_rope_head_dim=_QK_ROPE,
qk_nope_head_dim=_QK_NOPE,
v_head_dim=_V_HEAD_DIM,
num_heads=_NUM_HEADS,
kv_b_proj=kv_b_proj,
_use_flashinfer_concat_mla_k=False,
)
impl._concat_k_nope_k_pe = MLACommonImpl._concat_k_nope_k_pe.__get__(impl)
monkeypatch.setattr(mla_attention.ops, "cp_gather_cache", lambda **_: None)
monkeypatch.setattr(mla_attention, "_get_kv_b_proj_input_dtype", lambda *_: None)
monkeypatch.setattr(
mla_attention,
"reorg_kvcache",
lambda kv_c, k_pe, **_: (kv_c[:_DCP_TOKENS].squeeze(1), k_pe[:_DCP_TOKENS]),
)
chunk = SimpleNamespace(
index=0,
padded_local_seq_lens=[_DCP_TOKENS],
local_context_lens_allranks=[[_DCP_TOKENS // _DCP] * _DCP],
padded_local_cu_seq_lens=torch.tensor([0, _DCP_TOKENS], device=device),
padded_local_token_to_seq=torch.zeros(
_DCP_TOKENS, dtype=torch.int32, device=device
),
local_starts=[0],
num_local_context_tokens=_DCP_TOKENS,
request_slice=slice(0, 1),
num_requests=1,
starts=torch.tensor([0], device=device),
num_context_tokens=_DCP_TOKENS,
max_seq_len=_DCP_TOKENS,
token_slice=slice(0, 4),
)
backend = _RecordingPrefillBackend()
attn_metadata = SimpleNamespace(
prefill=SimpleNamespace(
prefill_backend=backend,
chunked_context=SimpleNamespace(
workspace=workspace,
chunks=[chunk],
empty_token_slices=False,
dcp_manager=SimpleNamespace(kv_gather=lambda dst, src: None),
),
q_data_type=q_data_type,
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
)
)
q = torch.randn(
(4, _NUM_HEADS, _QK_NOPE + _QK_ROPE), dtype=torch.bfloat16, device=device
)
out, lse = MLACommonImpl._context_parallel_compute_prefill_context(
impl,
q,
torch.empty(0, device=device),
attn_metadata,
k_scale=torch.ones(1, device=device),
dcp_world_size=_DCP,
fused_mla_kv_concat_fn=fused_mla_kv_concat_fn,
)
return backend, out, lse
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@torch.inference_mode()
def test_dcp_context_fused_pack_matches_generic_tail(
monkeypatch, kv_cache_dtype: str
) -> None:
"""Under DCP the layer's fused K/V pack must feed the prefill backend exactly
what the generic impl's cast + split + concat tail feeds it."""
q_data_type = (
current_platform.fp8_dtype() if kv_cache_dtype == "fp8" else torch.bfloat16
)
layer = SimpleNamespace(qk_nope_head_dim=_QK_NOPE, v_head_dim=_V_HEAD_DIM)
fused_fn = MultiHeadLatentAttention._fused_mla_kv_concat.__get__(layer)
backend_ref, ref_out, ref_lse = _run_dcp_context_loop(
monkeypatch, q_data_type=q_data_type, fused_mla_kv_concat_fn=None
)
backend_fused, fused_out, fused_lse = _run_dcp_context_loop(
monkeypatch, q_data_type=q_data_type, fused_mla_kv_concat_fn=fused_fn
)
assert len(backend_ref.calls) == len(backend_fused.calls) == 1
for name, fused_t, ref_t in zip(
("q", "k", "v"), backend_fused.calls[0], backend_ref.calls[0], strict=True
):
torch.testing.assert_close(
fused_t, ref_t, atol=0, rtol=0, msg=lambda m, n=name: f"{n} differs: {m}"
)
torch.testing.assert_close(fused_out, ref_out, atol=0, rtol=0)
torch.testing.assert_close(fused_lse, ref_lse, atol=0, rtol=0)
@@ -1,31 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import nn
from vllm.models.kimi_k3.nvidia.model import KimiDecoderLayer
class _ReturningSharedKDA(nn.Module):
def __init__(self, result: torch.Tensor) -> None:
super().__init__()
self.result = result
def forward(
self, hidden_states: torch.Tensor, positions: torch.Tensor
) -> torch.Tensor:
return self.result
def test_low_rank_kda_caller_returns_projection_storage_directly() -> None:
hidden_states = torch.randn(4, 8)
projected = torch.randn_like(hidden_states)
layer = object.__new__(KimiDecoderLayer)
nn.Module.__init__(layer)
layer.self_attn = _ReturningSharedKDA(projected)
output = layer._run_self_attn(torch.arange(4), hidden_states)
assert output is projected
assert output.data_ptr() == projected.data_ptr()
-504
View File
@@ -1,504 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
import contextlib
import json
import os
import random
import shutil
import socket
import subprocess
import time
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
from uuid import uuid4
import pytest
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.models.utils import check_logprobs_close
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
# Official Kimi-K3 config with 8 layers and no weights.
MODEL = "riverclouds/Kimi-K3-8L-dummy"
# The recipe's DSpark draft, reading target layers that exist in MODEL.
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
NUM_TURNS = 3
FIRST_PROMPT_BLOCKS = 3
UNALIGNED_TAIL = 37
TURN_TOKENS = 777
MAX_TOKENS = 8
NUM_LOGPROBS = 5
STARTUP_TIMEOUT = 1200
BASE_ARGS = [
"--trust-remote-code",
"--language-model-only",
"--load-format",
"dummy",
"--no-enable-flashinfer-autotune",
"--enable-prompt-tokens-details",
# Default max_num_seqs OOMs one GPU during warmup.
"--max-num-seqs",
"8",
]
pytestmark = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
)
SPEC_CONFIG = {
"method": "dspark",
"model": DRAFT,
"num_speculative_tokens": 7,
"draft_sample_method": "probabilistic",
"rejection_sample_method": "block",
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
MOONCAKE = {
"kv_connector": "MooncakeStoreConnector",
"kv_role": "kv_both",
"kv_load_failure_policy": "recompute",
"kv_connector_extra_config": {
"load_async": True,
"lookup_async": True,
"enable_cross_layers_blocks": False,
},
}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"cpu_bytes_to_use_per_rank": 32 << 30,
"lazy_offload": False,
},
}
NIXL_OFFLOAD = {
"kv_connector": "MultiConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
}
@dataclass(frozen=True)
class Mode:
# A finer prefix_match_unit enables partial hits inside a Mamba block.
prefix_match_unit: int | None
spec: bool
@property
def name(self) -> str:
name = "block" if self.prefix_match_unit is None else "partial"
return f"{name}-dspark" if self.spec else name
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
@dataclass(frozen=True)
class Instance:
tp: int = 1
dp: int = 1
dcp: int = 1
kv_config: dict | None = None
prefix_caching: bool = True
@property
def num_gpus(self) -> int:
return self.tp * self.dp
def args(self, mode: Mode) -> list[str]:
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
if mode.spec:
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
if self.dp > 1:
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
if self.dcp > 1:
args += ["--decode-context-parallel-size", str(self.dcp)]
if self.kv_config is not None:
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
if not self.prefix_caching:
return args + ["--no-enable-prefix-caching"]
args.append("--enable-prefix-caching")
if mode.prefix_match_unit is not None:
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
return args
@dataclass(frozen=True)
class Deployment:
decode: Instance
prefill: Instance | None = None
# Reset GPU prefix caches each turn so hits must come from CPU.
offload: bool = False
@property
def instances(self) -> tuple[Instance, ...]:
return (self.prefill, self.decode) if self.prefill else (self.decode,)
@property
def num_gpus(self) -> int:
return sum(i.num_gpus for i in self.instances)
DEPLOYMENTS = {
"plain": Deployment(Instance()),
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
"dcp2": Deployment(Instance(tp=2, dcp=2)),
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
"pd-tp2-dep2": Deployment(
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
prefill=Instance(tp=2, kv_config=NIXL),
),
"pd-offload": Deployment(
Instance(kv_config=NIXL_OFFLOAD),
prefill=Instance(kv_config=NIXL_OFFLOAD),
offload=True,
),
}
def _known_failure(name: str, mode: Mode) -> str | None:
deployment = DEPLOYMENTS[name]
if mode.spec and deployment.decode.dcp > 1:
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
return None
def _case(name: str, mode: Mode):
marks = []
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
marks += multi_gpu_marks(num_gpus=num_gpus)
if reason := _known_failure(name, mode):
marks.append(pytest.mark.xfail(reason=reason, strict=True))
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
def _block_size(url: str) -> int:
response = requests.get(f"{url}/metrics", timeout=30)
response.raise_for_status()
for family in text_string_to_metric_families(response.text):
for sample in family.samples:
if sample.name == "vllm:cache_config_info":
return lcm(
*(
int(value)
for key in ("block_size", "mamba_block_size")
if (value := sample.labels.get(key, "None")) != "None"
)
)
raise AssertionError("missing vllm:cache_config_info metric")
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
body = {
"model": MODEL,
"prompt": prompt,
"max_tokens": MAX_TOKENS,
"temperature": 0,
"logprobs": NUM_LOGPROBS,
"return_tokens_as_token_ids": True,
"cache_salt": salt,
**extra,
}
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
response.raise_for_status()
return response.json()
def _pd_complete(
prefill_url: str, decode_url: str, prompt: list[int], salt: str
) -> tuple[dict, dict]:
remote_decode = {
"do_remote_decode": True,
"do_remote_prefill": False,
"remote_engine_id": None,
"remote_block_ids": None,
"remote_host": None,
"remote_port": None,
}
prefill = _complete(
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
)
decode = _complete(
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
)
return prefill, decode
def _cached(response: dict) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
def _token_id(token: str) -> int:
return int(token.removeprefix("token_id:"))
def _tokens_text_logprobs(response: dict):
choice = response["choices"][0]
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
top = [
{_token_id(t): lp for t, lp in d.items()}
for d in choice["logprobs"]["top_logprobs"]
]
return ids, choice["text"], top
def _reset_gpu_prefix_cache(url: str) -> None:
deadline = time.monotonic() + 60
while True:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
if response.json()["success"]:
return
assert time.monotonic() < deadline, "GPU cache reset timed out"
time.sleep(0.1)
@dataclass(frozen=True)
class Turn:
prompt_len: int
cached: int
expected_cached: int
recompute_cached: int
output: tuple
recompute: tuple
@property
def matches_recompute(self) -> bool:
return self.output[0] == self.recompute[0]
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
"""A turn must reuse the previous prompt up to its last checkpoint.
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
"""
if not prev_prompt_len:
return 0
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
return max(checkpoint - hit_unit, 0) if spec else checkpoint
@contextlib.contextmanager
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
with contextlib.ExitStack() as stack:
# Sequential shutdown waits on memory still held by sibling servers.
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
next_gpu = 0
for instance in deployment.instances:
env = {}
if deployment.offload:
# Enables /reset_prefix_cache.
env["VLLM_SERVER_DEV_MODE"] = "1"
if deployment.prefill is not None:
gpus = range(next_gpu, next_gpu + instance.num_gpus)
next_gpu += instance.num_gpus
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
servers.append(
RemoteOpenAIServer(
MODEL,
instance.args(mode),
env_dict=env,
max_wait_seconds=STARTUP_TIMEOUT,
)
)
yield servers
def _run_conversation(
deployment: Deployment,
servers: list[RemoteOpenAIServer],
mode: Mode,
) -> list[Turn]:
# Under P/D the prefiller computes the prompt, so hits are read from it.
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
block_size = _block_size(compute_url)
hit_unit = mode.prefix_match_unit or block_size
rng = random.Random(0)
def new_tokens(n: int) -> list[int]:
return [rng.randint(1000, 150000) for _ in range(n)]
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
prev_len = 0
turns = []
for turn in range(NUM_TURNS):
if deployment.offload:
for server in servers:
_reset_gpu_prefix_cache(server.url_root)
if deployment.prefill is not None:
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
else:
computed = output = _complete(decode_url, prompt, "conv")
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
turns.append(
Turn(
prompt_len=len(prompt),
cached=_cached(computed),
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
recompute_cached=_cached(recompute),
output=_tokens_text_logprobs(output),
recompute=_tokens_text_logprobs(recompute),
)
)
prev_len = len(prompt)
prompt = prompt + new_tokens(TURN_TOKENS)
return turns
def _format(turns: list[Turn]) -> str:
rows = ["turn prompt cached expected matches_recompute"]
rows += [
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
f"{t.matches_recompute}"
for i, t in enumerate(turns)
]
return "\n".join(rows)
def _check_resend_reuses_prompt_checkpoint(
deployment: Deployment, prompt_len: int
) -> None:
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
mode = Mode(128, True)
salt = str(uuid4())
rng = random.Random(0)
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
with _serve(deployment, mode) as servers:
url = servers[0].url_root
cold = _complete(url, prompt, salt, max_tokens=1)
if deployment.offload:
_reset_gpu_prefix_cache(url)
resend = _complete(url, prompt, salt, max_tokens=1)
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
expected = (prompt_len // 128 - 1) * 128
actual = _cached(resend)
assert _cached(cold) == _cached(recompute) == 0
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
check_logprobs_close(
outputs_0_lst=[_tokens_text_logprobs(recompute)],
outputs_1_lst=[_tokens_text_logprobs(resend)],
name_0="recompute",
name_1="resend",
)
@pytest.mark.parametrize(
"prompt_len",
[
# Blocks=6144, PMU=128. P=7040 is PMU-aligned but not block-aligned:
# ensure lookup reads the cached attention proof at 7040 despite
# the final hit limit of 7039, then applies EAGLE's 128-token drop.
pytest.param(7040, id="eagle-proof-scan"),
# Blocks=6144, PMU=128. P=7296 is an exact PMU multiple:
# ensure the Mamba checkpoint is 7168, not 7040 from applying both
# the P - 1 cap and EAGLE's 128-token drop.
pytest.param(7296, id="eagle-double-cap"),
],
)
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
@pytest.fixture
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
"""Run a Mooncake master and point the servers' store config at it."""
if shutil.which("mooncake_master") is None:
pytest.skip("mooncake_master is not installed")
port = get_open_port()
log_path = tmp_path / "mooncake_master.log"
with open(log_path, "w") as log:
master = subprocess.Popen(
[
"mooncake_master",
f"--port={port}",
f"--metrics_port={get_open_port()}",
"--enable_metric_reporting=false",
"--default_kv_lease_ttl=60000",
],
stdout=log,
stderr=subprocess.STDOUT,
)
try:
deadline = time.monotonic() + 60
while True:
assert master.poll() is None, log_path.read_text()
with (
contextlib.suppress(OSError),
socket.create_connection(("127.0.0.1", port)),
):
break
assert time.monotonic() < deadline, "Mooncake master did not start"
time.sleep(0.1)
config_path = tmp_path / "mooncake.json"
config_path.write_text(
json.dumps(
{
"metadata_server": "P2PHANDSHAKE",
"master_server_address": f"127.0.0.1:{port}",
"protocol": "rdma",
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
"global_segment_size": "4GB",
"local_buffer_size": "1GB",
}
)
)
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
yield
finally:
master.kill()
master.wait()
@pytest.mark.usefixtures("mooncake_store")
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
remain reusable from the store after a GPU cache reset."""
_check_resend_reuses_prompt_checkpoint(
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
)
@pytest.mark.parametrize(
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
)
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
deployment = DEPLOYMENTS[name]
with _serve(deployment, mode) as servers:
turns = _run_conversation(deployment, servers, mode)
table = _format(turns)
print(f"\n{name}-{mode.name}\n{table}")
assert all(t.recompute_cached == 0 for t in turns), table
assert all(t.cached >= t.expected_cached for t in turns), table
# Dummy weights differ across TP sizes.
if len({i.tp for i in deployment.instances}) == 1:
check_logprobs_close(
outputs_0_lst=[t.recompute for t in turns],
outputs_1_lst=[t.output for t in turns],
name_0="recompute",
name_1=name,
)
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Align-mode RecoverSSM state columns at recurrent block boundaries.
A speculative window whose accepted tokens end exactly on a recurrent block
boundary must keep its state in the block holding the last committed token.
The following block can still be unallocated, so selecting it would drop the
commit while the bookkeeping recorded the state as advanced.
"""
import pytest
import torch
from vllm.models.kimi_k3.nvidia.ops.recoverssm import _prepare_commit_plan_kernel
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.worker.gpu.model_states.recoverssm import (
_postprocess_recoverssm_align_kernel,
)
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="The RecoverSSM kernels require a CUDA-alike device.",
)
BLOCK_SIZE = 2048
SPEC_QUERY_LEN = 4
BLOCKS = [100, 101, 102, 103, 104, 105]
# (computed before the step, committed tokens, allocated block columns,
# final column, boundary column or None)
CASES = [
# 8,188 + 4 ends exactly on the 8,192 boundary; column 4 is unallocated.
(8188, 4, 4, 3, 3),
# The window crosses the boundary: state moves on, checkpoint stays behind.
(8190, 4, 5, 4, 3),
# The window stays inside column 3.
(8180, 4, 4, 3, None),
# The first token after a boundary belongs to the new column.
(8192, 1, 5, 4, None),
# Single-token commit ending on the first boundary.
(2047, 1, 1, 0, 0),
]
@pytest.mark.parametrize(
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
)
def test_commit_plan_keeps_boundary_state_in_last_computed_block(
num_computed, commit_len, allocated, final_col, boundary_col
):
device = current_platform.device_type
columns = [
block if col < allocated else NULL_BLOCK_ID for col, block in enumerate(BLOCKS)
]
block_table = torch.tensor([columns], dtype=torch.int32, device=device)
num_accepted = torch.tensor([commit_len], dtype=torch.int32, device=device)
state_indices = torch.tensor(
[BLOCKS[(num_computed - 1) // BLOCK_SIZE]], dtype=torch.int32, device=device
)
query_start_loc = torch.tensor(
[0, SPEC_QUERY_LEN], dtype=torch.int32, device=device
)
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
commit_lens, final_indices, boundary_indices, recovery_lens = (
torch.full((1,), -7, dtype=torch.int32, device=device) for _ in range(4)
)
_prepare_commit_plan_kernel[(1,)](
num_accepted,
None,
state_indices,
query_start_loc,
block_table,
computed,
commit_lens,
final_indices,
boundary_indices,
recovery_lens,
NULL_BLOCK_ID,
BLOCK_SIZE,
block_table.shape[1],
num_accepted.stride(0),
0,
state_indices.stride(0),
query_start_loc.stride(0),
block_table.stride(0),
block_table.stride(1),
computed.stride(0),
SPEC_QUERY_LEN=SPEC_QUERY_LEN,
num_warps=1,
)
assert commit_lens.item() == commit_len
assert final_indices.item() == BLOCKS[final_col]
expected_boundary = NULL_BLOCK_ID if boundary_col is None else BLOCKS[boundary_col]
assert boundary_indices.item() == expected_boundary
if boundary_col is not None:
next_boundary = (num_computed // BLOCK_SIZE + 1) * BLOCK_SIZE
assert recovery_lens.item() == next_boundary - num_computed
@pytest.mark.parametrize(
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
)
def test_postprocess_records_the_committed_state_column(
num_computed, commit_len, allocated, final_col, boundary_col
):
device = current_platform.device_type
idx_mapping = torch.tensor([2], dtype=torch.int32, device=device)
num_sampled = torch.tensor([commit_len], dtype=torch.int32, device=device)
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
state_idx = torch.full((4,), -7, dtype=torch.int32, device=device)
num_accepted = torch.full((4,), -7, dtype=torch.int32, device=device)
_postprocess_recoverssm_align_kernel[(1,)](
idx_mapping,
num_sampled,
None,
computed,
state_idx,
num_accepted,
MAMBA_BLOCK_SIZE=BLOCK_SIZE,
BLOCK_TABLE_WIDTH=len(BLOCKS),
)
assert state_idx[2].item() == final_col
assert num_accepted[2].item() == 1
assert state_idx[[0, 1, 3]].tolist() == [-7, -7, -7]
@@ -1,461 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from types import MethodType, SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from torch import nn
from vllm.config import ParallelConfig
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.models.common.ops import sequence_parallel as sp_ops
from vllm.models.kimi_k3.nvidia import model as kimi_model
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
from vllm.platforms import current_platform
class _IdentityNorm(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
self.variance_epsilon = 1e-5
def forward(
self,
hidden_states: torch.Tensor,
residual: torch.Tensor | None = None,
):
if residual is None:
return hidden_states
return hidden_states, residual
class _RecordingMoE(nn.Module):
def __init__(self) -> None:
super().__init__()
self.num_tokens = 0
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
self.num_tokens = hidden_states.shape[0]
return hidden_states
class _Projection(nn.Module):
def __init__(self, hidden_size: int = 2) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.ones(1, hidden_size),
requires_grad=False,
)
class _SequenceParallelMTPBlock:
use_sequence_parallel = True
def __call__(
self,
*,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
):
assert residual is None
return hidden_states * 2, None, hidden_states * 3
def _mock_sequence_parallel_collectives(monkeypatch):
monkeypatch.setattr(
kimi_model,
"sp_reduce_scatter",
lambda tensor: tensor.chunk(2, dim=0)[0],
)
monkeypatch.setattr(
kimi_model,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_model,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
@pytest.mark.parametrize(
("num_tokens", "is_padding", "tp_rank", "expected"),
[
(1, None, 0, [False]),
(1, None, 1, [True]),
(5, None, 2, [False, True]),
(5, None, 3, [True, True]),
(5, [False, True, False, False, False], 0, [False, True]),
],
)
def test_sp_padding_mask_marks_added_rows(
monkeypatch,
num_tokens: int,
is_padding: list[bool] | None,
tp_rank: int,
expected: list[bool],
):
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
hidden_states = torch.empty(num_tokens, 2)
padding = torch.tensor(is_padding) if is_padding is not None else None
actual = sp_ops.sp_padding_mask(padding, hidden_states)
torch.testing.assert_close(actual, torch.tensor(expected))
@pytest.mark.parametrize(
("data_parallel_size", "expected"),
[
(1, False),
(2, True),
],
)
def test_moe_sequence_parallel_requires_data_parallel(
monkeypatch,
data_parallel_size: int,
expected: bool,
):
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
parallel_config = ParallelConfig(
tensor_parallel_size=2,
data_parallel_size=data_parallel_size,
enable_expert_parallel=True,
all2all_backend="allgather_reducescatter",
)
assert parallel_config.use_sequence_parallel_moe is expected
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = False
layer.use_sequence_parallel = True
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
positions = torch.arange(3)
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
hidden_states = kimi_model.sp_shard(full_hidden_states)
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=None,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
hidden_states, prefix_sum, residual = layer(
positions=positions,
hidden_states=hidden_states,
residual=residual,
)
assert prefix_sum is None
assert hidden_states.shape == residual.shape == (2, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
layer = object.__new__(kimi_model.KimiDecoderLayer)
nn.Module.__init__(layer)
layer.use_attn_res = True
layer.use_sequence_parallel = True
layer.prev_valid_blocks = 0
layer.block_write_idx = 0
layer.is_block_write_layer = False
layer.input_layernorm = _IdentityNorm()
layer.post_attention_layernorm = _IdentityNorm()
layer.self_attention_res_norm = _IdentityNorm()
layer.mlp_res_norm = _IdentityNorm()
layer.self_attention_res_proj = _Projection()
layer.mlp_res_proj = _Projection()
layer.mlp = _RecordingMoE()
layer._run_self_attn = MethodType(
lambda self, positions, hidden_states: hidden_states,
layer,
)
_mock_sequence_parallel_collectives(monkeypatch)
monkeypatch.setattr(
kimi_model,
"attn_res",
lambda prefix_sum, hidden_states, *args, **kwargs: (
prefix_sum if hidden_states is None else prefix_sum + hidden_states
),
)
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
block_residual = torch.zeros(2, 1, 2)
hidden_states, prefix_sum, block_residual = layer(
positions=torch.arange(3),
hidden_states=None,
prefix_sum=prefix_sum,
residual=block_residual,
)
assert hidden_states.shape == prefix_sum.shape == (2, 2)
assert block_residual.shape == (2, 1, 2)
assert layer.mlp.num_tokens == 2
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
nn.Module.__init__(layer)
layer.enorm = _IdentityNorm()
layer.hnorm = _IdentityNorm()
layer.eh_proj = nn.Identity()
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
object.__setattr__(
layer,
"shared_head",
SimpleNamespace(norm=final_norm),
)
monkeypatch.setattr(
kimi_mtp,
"fused_mtp_input",
lambda positions, inputs_embeds, *args: inputs_embeds,
)
monkeypatch.setattr(
kimi_mtp,
"sp_shard",
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
)
monkeypatch.setattr(
kimi_mtp,
"sp_all_gather",
lambda tensor: torch.cat([tensor, tensor], dim=0),
)
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
logits_hidden_states, hidden_states = layer(
input_ids=torch.zeros(3, dtype=torch.long),
positions=torch.arange(3),
previous_hidden_states=torch.zeros_like(inputs_embeds),
inputs_embeds=inputs_embeds,
)
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
expected_hidden_states = torch.cat(
[sharded_states * 5, sharded_states * 5],
dim=0,
)[:3]
torch.testing.assert_close(hidden_states, expected_hidden_states)
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
final_norm.assert_called_once()
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
@pytest.mark.parametrize(
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
[
(True, True, True, 8, True),
(False, True, True, 8, False), # opt-in only
(True, False, True, 8, False), # replication only exists under SP
(True, True, False, 8, False), # FusedMoE path owns the reduction
(True, True, True, 1, False), # nothing to shard
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
],
)
def test_shard_sequence_parallel_mlp_gating(
monkeypatch,
enabled: bool,
use_sequence_parallel: bool,
eligible: bool,
tp_size: int,
expected: bool,
):
monkeypatch.setattr(kimi_model.envs, "VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", enabled)
monkeypatch.setattr(
kimi_model, "get_tensor_model_parallel_world_size", lambda: tp_size
)
assert (
kimi_model.shard_sequence_parallel_mlp(
hidden_size=7168,
intermediate_size=6144,
use_sequence_parallel=use_sequence_parallel,
eligible=eligible,
)
is expected
)
@pytest.mark.parametrize(
("moe_backend", "all2all_backend", "expected"),
[
("deep_gemm_mega_moe", "naive", True),
("auto", "deepep_v2", True),
("auto", "deepep_low_latency", False),
],
)
def test_shared_expert_sharding_backend_gate(
moe_backend: str,
all2all_backend: str,
expected: bool,
):
vllm_config = SimpleNamespace(
kernel_config=SimpleNamespace(moe_backend=moe_backend),
parallel_config=SimpleNamespace(all2all_backend=all2all_backend),
)
assert kimi_model.can_shard_sequence_parallel_shared_expert(vllm_config) is expected
def test_sharded_sequence_parallel_mlp_matches_replicated(default_vllm_config):
"""Sharded SP MLP must reproduce the replicated result for every token.
Each rank owns a *disjoint* token shard, so a weight shard alone cannot
finish a rank's own tokens: the ranks must gather the full token set,
compute partial sums over their intermediate shard, and reduce-scatter.
Splicing per-rank feature slices together instead silently mixes different
tokens and produces plausible-looking garbage.
"""
tp_size, hidden, intermediate, tokens_per_rank = 4, 16, 12, 3
torch.manual_seed(0)
num_tokens = tp_size * tokens_per_rank
x = torch.randn(num_tokens, hidden)
gate_weight = torch.randn(intermediate, hidden)
up_weight = torch.randn(intermediate, hidden)
down_weight = torch.randn(hidden, intermediate)
act_fn = SiluAndMul()
replicated = act_fn(x @ torch.cat([gate_weight, up_weight]).T) @ down_weight.T
shard = intermediate // tp_size
# Every rank all-gathers the full token set, then computes its partial.
partials = [
act_fn(
x
@ torch.cat(
[
gate_weight[r * shard : (r + 1) * shard],
up_weight[r * shard : (r + 1) * shard],
]
).T
)
@ down_weight[:, r * shard : (r + 1) * shard].T
for r in range(tp_size)
]
reduced = torch.stack(partials).sum(0)
# Reduce-scatter: rank r keeps only its own token shard.
for r in range(tp_size):
mine = reduced[r * tokens_per_rank : (r + 1) * tokens_per_rank]
expected = replicated[r * tokens_per_rank : (r + 1) * tokens_per_rank]
torch.testing.assert_close(mine, expected, atol=1e-5, rtol=1e-5)
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
expected = torch.cat([hidden_states, hidden_states])
custom_all_gather = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_all_gather=custom_all_gather,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
output = sp_ops.sp_all_gather(hidden_states)
torch.testing.assert_close(output, expected)
custom_all_gather.assert_called_once_with(hidden_states)
fallback.assert_not_called()
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
custom_reduce_scatter = Mock(return_value=expected)
device_communicator = SimpleNamespace(
custom_reduce_scatter=custom_reduce_scatter,
)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=device_communicator),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
output = sp_ops.sp_reduce_scatter(hidden_states)
torch.testing.assert_close(output, expected)
padded = custom_reduce_scatter.call_args.args[0]
assert padded.shape == (4, 2)
torch.testing.assert_close(padded[:3], hidden_states)
torch.testing.assert_close(padded[3], torch.zeros(2))
fallback.assert_not_called()
@pytest.mark.parametrize("shape", [(3,), (3, 2, 2)])
def test_sp_shard_pads_only_the_token_axis(monkeypatch, shape):
hidden_states = torch.arange(math.prod(shape), dtype=torch.float32).view(shape)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: 1)
output = sp_ops.sp_shard(hidden_states)
padding = hidden_states.new_zeros((1, *shape[1:]))
expected = torch.cat([hidden_states, padding])[2:]
torch.testing.assert_close(output, expected)
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
monkeypatch.setattr(
sp_ops,
"get_tp_group",
lambda: SimpleNamespace(device_communicator=None),
)
monkeypatch.setattr(
sp_ops,
"get_tensor_model_parallel_world_size",
lambda: 2,
)
all_gather = Mock(return_value=hidden_states)
reduce_scatter = Mock(return_value=hidden_states)
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
monkeypatch.setattr(
sp_ops,
"tensor_model_parallel_reduce_scatter",
reduce_scatter,
)
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
all_gather.assert_called_once_with(hidden_states, 0)
reduce_scatter.assert_called_once_with(hidden_states, 0)
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models import gemma
from vllm.model_executor.models.gemma3n import (
Gemma3nTextModel,
_kv_sharing_weights_mapper,
)
from vllm.model_executor.models.gemma4 import (
Gemma4ForCausalLM,
_gemma4_layer_weights_mapper,
)
from vllm.model_executor.models.gemma4_dspark import (
Gemma4DSparkForCausalLM,
Gemma4DSparkModel,
)
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize(
"enabled,has_weights,with_markov",
[
pytest.param(True, True, False, id="hidden-only"),
pytest.param(True, True, True, id="with-markov"),
pytest.param(True, False, True, id="missing-weights"),
pytest.param(False, True, True, id="disabled-head"),
],
)
def test_gemma4_dspark_loads_confidence_head(
monkeypatch, enabled, has_weights, with_markov
) -> None:
"""Use checkpoint confidence parameters, or disable an unavailable head."""
config = SimpleNamespace(
vocab_size=64,
hidden_size=8,
target_layer_ids=[0, 1],
num_hidden_layers=0,
rms_norm_eps=1e-6,
markov_rank=4,
enable_confidence_head=enabled,
confidence_head_with_markov=with_markov,
)
vllm_config = SimpleNamespace(
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace(
draft_model_config=SimpleNamespace(hf_config=config),
),
)
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
width = config.hidden_size + (config.markov_rank if with_markov else 0)
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
loaded = model.load_weights(
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
if has_weights
else []
)
if not (enabled and has_weights):
assert model.model.confidence_head is None
assert not loaded
return
assert loaded == {
"model.confidence_head.proj.weight",
"model.confidence_head.proj.bias",
}
assert model.model.confidence_head.proj.weight.dtype == torch.float32
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
"""A physical LM head must load after checkpoint-driven untying."""
class StubGemmaModel(torch.nn.Module):
def __init__(self, *, vllm_config, prefix):
super().__init__()
self.embed_tokens = VocabParallelEmbedding(4, 2)
self.make_empty_intermediate_tensors = None
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
config = SimpleNamespace(
vocab_size=4,
hidden_size=2,
tie_word_embeddings=False,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
)
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
embedding_weight = torch.full((4, 2), 1.0)
lm_head_weight = torch.full((4, 2), 2.0)
loaded = model.load_weights(
[
("model.embed_tokens.weight", embedding_weight),
("lm_head.weight", lm_head_weight),
]
)
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
@pytest.mark.cpu_test
def test_gemma4_attention_mapper() -> None:
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
layers also load K as the V shard, and leave any v_proj such a checkpoint
ships unmapped so it fails the load rather than silently overwriting V;
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
still ship for them."""
config = SimpleNamespace(
num_hidden_layers=3,
num_kv_shared_layers=1,
attention_k_eq_v=True,
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
)
weights = [
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
for i in range(3)
for tensor in ("q_proj", "k_proj", "k_norm")
] + [
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
]
mapper = _gemma4_layer_weights_mapper(config)
mapped = list(mapper.apply(weights))
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
("model.layers.0.self_attn.qkv_proj.weight", "q"),
("model.layers.0.self_attn.qkv_proj.weight", "k"),
("model.layers.0.self_attn.k_norm.weight", None),
("model.layers.1.self_attn.qkv_proj.weight", "q"),
("model.layers.1.self_attn.qkv_proj.weight", "k"),
("model.layers.1.self_attn.qkv_proj.weight", "v"),
("model.layers.1.self_attn.k_norm.weight", None),
("model.layers.2.self_attn.q_proj.weight", None),
("model.layers.0.mlp.gate_up_proj.weight", 1),
("model.layers.1.self_attn.v_proj.weight", None),
]
k_weight, v_weight = weights[4][1], mapped[5][1]
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
@pytest.mark.cpu_test
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
"""The text-only path reuses the conditional wrapper's checkpoint naming,
so fused and per-expert tensors reach the experts under `model.*`."""
prefix = "model.language_model.layers.0."
weights = [
(prefix + name, torch.empty(0))
for name in (
"experts.gate_up_proj",
"experts.3.down_proj.weight_packed",
"router.per_expert_scale",
)
]
mapped = [
(name, getattr(w, "shard_id", None))
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
]
assert mapped == [
("model.layers.0.experts.gate_up_proj", None),
("model.layers.0.experts.3.down_proj.weight_packed", None),
("model.layers.0.router.per_expert_scale", None),
]
@pytest.mark.cpu_test
def test_gemma3n_kv_shared_layer_mapper() -> None:
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
q_proj and drop the redundant K/V tensors original checkpoints ship."""
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
weights = [
(f"layers.{i}.self_attn.{tensor}.weight", torch.empty(0))
for i in (1, 3)
for tensor in ("q_proj", "k_proj", "v_proj", "k_norm", "o_proj")
]
mapped = [
(name, getattr(weight, "shard_id", None))
for name, weight in mapper.apply(weights)
]
assert mapped == [
("layers.1.self_attn.qkv_proj.weight", "q"),
("layers.1.self_attn.qkv_proj.weight", "k"),
("layers.1.self_attn.qkv_proj.weight", "v"),
("layers.1.self_attn.k_norm.weight", None),
("layers.1.self_attn.o_proj.weight", None),
("layers.3.self_attn.q_proj.weight", None),
("layers.3.self_attn.o_proj.weight", None),
]
@pytest.mark.parametrize("model", MODELS)
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
with monkeypatch.context() as m:
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
with vllm_runner(
model,
load_format="dummy",
) as llm:
if model == "google/gemma-3-4b-it":
normalizers = llm.llm.collective_rpc(
lambda self: (
self.model_runner.model.language_model.model.normalizer.cpu().item()
) # noqa: E501
)
config = llm.llm.llm_engine.model_config.hf_config.text_config
else:
normalizers = llm.llm.collective_rpc(
lambda self: self.model_runner.model.model.normalizer.cpu().item()
)
config = llm.llm.llm_engine.model_config.hf_config
assert np.allclose(normalizers, config.hidden_size**0.5, rtol=2e-3)
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import GraniteConfig
from vllm.model_executor.models.granite import granite_layer_attn_params
from ...utils import check_logprobs_close
MODELS = [
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
"ibm-granite/granite-swash-2b",
"ibm-granite/granite-swash-3b-a600m",
]
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.cpu_model
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(model, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
def test_granite_swa_features_are_off_without_swa_config():
"""A plain Granite config must not pick up sliding windows or sinks."""
config = GraniteConfig(num_hidden_layers=4)
theta = config.rope_parameters["rope_theta"]
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
(None, theta, False)
] * 4
@pytest.mark.parametrize(
"attention_sinks, expected_sink", [(None, True), (False, False)]
)
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
config = GraniteConfig(
num_hidden_layers=3,
sliding_window=128,
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
layer_rope_theta=[10000.0, 0.0, 1000000.0],
**kwargs,
)
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
(None, 10000.0, expected_sink),
(128, 0.0, expected_sink),
(128, 1000000.0, expected_sink),
]
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from ....utils import large_gpu_test
from ...utils import check_logprobs_close
MODELS = [
"microsoft/Phi-3.5-MoE-instruct",
]
def test_phimoe_routing_function():
from vllm.model_executor.models.phimoe import phimoe_routing_function
test_case = {
0: {
"hidden_states": torch.tensor(
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
).view(4, 2),
"gating_output": torch.tensor(
[0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
),
"topk": 2,
"renormalize": False,
},
1: {
"hidden_states": torch.tensor(
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
).view(4, 2),
"gating_output": torch.tensor(
[0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
),
"topk": 2,
"renormalize": False,
},
}
ground_truth = {
0: {
"topk_weights": torch.tensor(
[1.0, 1.0], dtype=torch.float32, requires_grad=False
),
"topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
},
1: {
"topk_weights": torch.tensor(
[0.5, 1.0], dtype=torch.float32, requires_grad=False
),
"topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
},
}
for test_id in test_case:
topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"])
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
# There is a known issue that triggers `AttributeError: 'DynamicCache'
# object has no attribute 'seen_tokens'` when running:
# `tests/models/language/generation/test_phimoe.py::test_models
# [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]`
# This issue is being investigated and tracked in:
# https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58
# It is platform-agnostic. Therefore, we skip this test on all platforms for now.
@pytest.mark.skip(
reason="Skipping due to known issue: "
"'DynamicCache' object has no attribute 'seen_tokens'. See: "
"https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 "
"for details.",
)
@pytest.mark.skipif(
condition=current_platform.is_cpu(),
reason="This test takes a lot time to run on CPU, "
"and vllm CI's disk space is not enough for this model.",
)
@large_gpu_test(min_gb=80)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(model, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
-43
View File
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pytest configuration for vLLM language generation tests."""
import gc
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
def pytest_sessionstart(session):
"""Configure ROCm-specific settings before test session starts."""
if current_platform.is_rocm():
torch.set_float32_matmul_precision("high")
@pytest.fixture(autouse=True)
def release_gpu_memory_between_tests():
"""Reclaim cached GPU memory between tests in this shard.
These tests start many vLLM/HF engines sequentially in one pytest
process; without explicit reclamation between tests, VRAM fragmentation
accumulates and later tests fail at engine startup ("Engine core
initialization failed").
"""
yield
gc.collect()
if torch.accelerator.is_available():
torch.accelerator.empty_cache()
if current_platform.is_rocm():
return
try:
wait_for_memory_to_settle()
except ValueError as e:
# Longer-lived (class/module-scoped) engine fixtures may legitimately
# still hold VRAM; nothing further to reclaim in that case.
logger.info("Failed to clean GPU memory: %s", e)
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import openai
import pytest
from tests.conftest import HfRunner
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
embeddings = []
for d in response.data:
assert isinstance(d.embedding, list)
embeddings.append(d.embedding)
return embeddings
def run_embedding_correctness_test(
hf_model: "HfRunner",
inputs: list[str],
vllm_outputs: Sequence[list[float]],
dimensions: int | None = None,
):
hf_outputs = hf_model.encode(inputs)
if dimensions:
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
def correctness_test_embed_models(
hf_runner,
vllm_runner,
model_info: EmbedModelInfo,
example_prompts,
vllm_extra_kwargs=None,
hf_model_callback=None,
):
pytest.skip("Debug only, ci prefers to use mteb test.")
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
vllm_extra_kwargs = vllm_extra_kwargs or {}
vllm_extra_kwargs["dtype"] = model_info.dtype
if model_info.hf_overrides is not None:
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
with vllm_runner(
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
with hf_runner(
model_info.name,
dtype=model_info.hf_dtype,
is_sentence_transformer=True,
) as hf_model:
if hf_model_callback is not None:
hf_model_callback(hf_model)
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
async def run_client_embeddings(
client: openai.AsyncOpenAI,
model_name: str,
queries: list[str],
instruction: str = "",
) -> list[list[float]]:
outputs = await client.embeddings.create(
model=model_name,
input=[instruction + q for q in queries],
)
return [data.embedding for data in outputs.data]
@@ -1,151 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from tests.models.utils import check_embeddings_close
from tests.utils import VLLM_PATH
from vllm import TokensPrompt
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-Embedding-0.6B"],
)
@torch.inference_mode
def test_decoder_token_embed_model_runner_v2(
hf_runner, vllm_runner, monkeypatch, model: str
):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
chunk_size = 10
n_prompt_tokens = [55, 56, 57]
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
max_model_len=128,
max_num_batched_tokens=chunk_size,
enforce_eager=True,
# `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner
enable_chunked_prefill=True,
enable_prefix_caching=True,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_embed(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
)
with hf_runner(
model,
auto_cls=AutoModel,
) as hf_model:
hf_outputs = []
for token_prompt in token_prompts:
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
input_ids = inputs["input_ids"]
output = hf_model.model(input_ids)
hf_outputs.append(output.last_hidden_state.cpu().float()[0])
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_output,
embeddings_1_lst=vllm_output,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
_RERANKER_HF_OVERRIDES = {
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
@torch.inference_mode
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
"""LAST-pooling score must not depend on whether the prompt is chunked.
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
not ``enforce_eager``), a query+document pair long enough to be split across
prefill chunks produced a corrupted last-token hidden state and thus a
wrong relevance score, while the same input scored correctly unchunked.
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
and so never exercised the compiled path.
"""
chat_template = _RERANKER_TEMPLATE.read_text()
query = "What organelle produces energy in the cell?"
# A long, matching document so query + doc exceeds the chunk size below.
document = (
"The mitochondria is the powerhouse of the cell. It generates most of "
"the cell chemical energy through oxidative phosphorylation. "
) * 400
def score_with(max_num_batched_tokens: int) -> float:
with vllm_runner(
model,
runner="pooling",
hf_overrides=_RERANKER_HF_OVERRIDES,
max_model_len=16384,
max_num_batched_tokens=max_num_batched_tokens,
enable_chunked_prefill=True,
enable_prefix_caching=False,
) as vllm_model:
return vllm_model.score(query, document, chat_template=chat_template)[0]
# Chunk size forces the prompt across several prefill chunks; the large
# budget keeps it in a single chunk as the reference.
chunked = score_with(2048)
unchunked = score_with(16384)
assert chunked == pytest.approx(unchunked, abs=5e-2), (
f"chunked score {chunked} diverged from unchunked {unchunked}"
)
@torch.inference_mode
def test_sequence_embed_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "Qwen/Qwen3-Embedding-0.6B"
chunk_size = 10
token_prompts = [[1024 + i for i in range(n)] for n in (25, 27)]
prompts = [TokensPrompt(prompt_token_ids=t) for t in token_prompts]
with hf_runner(model, auto_cls=AutoModel) as hf_model:
hf_outputs = []
for token_prompt in token_prompts:
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
output = hf_model.model(inputs["input_ids"])
embedding = torch.nn.functional.normalize(
output.last_hidden_state.float()[0, -1], dim=0
)
hf_outputs.append(embedding.cpu().tolist())
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="embed"),
max_model_len=64,
max_num_batched_tokens=chunk_size,
max_num_seqs=2,
gpu_memory_utilization=0.25,
enforce_eager=True,
enable_chunked_prefill=True,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,113 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
@pytest.mark.parametrize(
"model",
["jason9693/Qwen2.5-1.5B-apeach"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# example_prompts is too short for testing prefix_caching
example_prompts = [s * 10 for s in example_prompts]
with vllm_runner(
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert cache_config.enable_prefix_caching
# First Run
vllm_model.classify(example_prompts)
# assert prefix_caching works
pooling_outputs = vllm_model.llm.encode(
example_prompts, pooling_task="classify"
)
for output in pooling_outputs:
assert output.num_cached_tokens > 0
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
assert torch.allclose(
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
)
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-Embedding-0.6B"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_embed_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
):
# example_prompts is too short for testing prefix_caching
example_prompts = [str(s).strip() * 10 for s in example_prompts]
with vllm_runner(
model,
runner="pooling",
max_model_len=None,
enable_prefix_caching=True,
) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert cache_config.enable_prefix_caching
# First Run
vllm_model.embed(example_prompts)
# assert prefix_caching works
pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
for output in pooling_outputs:
assert output.num_cached_tokens > 0
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
with hf_runner(
model,
is_sentence_transformer=True,
) as hf_model:
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
@pytest.mark.parametrize(
"model",
[
"intfloat/e5-small",
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", # is_causal == False
"papluca/xlm-roberta-base-language-detection",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_non_causal_models(
hf_runner, vllm_runner, example_prompts, model: str, dtype: str
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_config = vllm_model.llm.llm_engine.vllm_config
cache_config = vllm_config.cache_config
assert not cache_config.enable_prefix_caching
@@ -1,212 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import httpx
import openai
import pytest
import torch
from ....utils import RemoteOpenAIServer
from .embed_utils import run_client_embeddings
MODEL_NAME = "BAAI/bge-m3"
MAX_MODEL_LEN = 512
# Example from https://huggingface.co/BAAI/bge-m3
sentences_1 = ["What is BGE M3?", "Definition of BM25"]
sentences_2 = [
"BGE M3 is an embedding model supporting dense retrieval, "
"lexical matching and multi-vector interaction.",
"BM25 is a bag-of-words retrieval function that ranks a set "
"of documents based on the query terms appearing in each document",
]
similarity_reference = [[0.6259, 0.3474], [0.3309, 0.6734]]
lexical_score_reference = [0.19554901123046875, 0.0]
colbert_score_reference = [0.7797, 0.4620]
SUPPORTED_TASKS = ["embed", "token_embed", "token_classify"]
@pytest.fixture(scope="module", params=SUPPORTED_TASKS)
def pooling_task(request):
yield request.param
@pytest.fixture(scope="module")
def server(pooling_task):
args = [
"--max-model-len",
str(MAX_MODEL_LEN),
"--hf-overrides",
'{"architectures": ["BgeM3EmbeddingModel"]}',
"--pooler-config.task",
pooling_task,
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_bge_m3_api_server_embedding(server, pooling_task):
client = server.get_async_client()
if pooling_task != "embed":
with pytest.raises(openai.InternalServerError):
await run_client_embeddings(
client,
MODEL_NAME,
sentences_1,
)
return
embeddings_list_1 = await run_client_embeddings(
client,
MODEL_NAME,
sentences_1,
)
embeddings_list_2 = await run_client_embeddings(
client,
MODEL_NAME,
sentences_2,
)
embeddings_1 = torch.tensor(embeddings_list_1)
embeddings_2 = torch.tensor(embeddings_list_2)
similarity = embeddings_1 @ embeddings_2.T
# reference values from BAAI/bge-m3 documentation
reference = torch.tensor(similarity_reference)
assert torch.allclose(similarity, reference, rtol=0.01)
async def tokenize(client: openai.AsyncOpenAI, sentences: list[str]) -> list[list[int]]:
futures = []
for sentence in sentences:
futures.append(
client.post(
"../tokenize",
body={"model": MODEL_NAME, "prompt": sentence},
cast_to=httpx.Response,
)
)
return [(await future).json()["tokens"] for future in futures]
async def sparse_embeddings(
client: openai.AsyncOpenAI, sentences: list[str]
) -> list[dict[int, float]]:
all_tokens = await tokenize(client, sentences)
result = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences, "task": "token_classify"},
cast_to=httpx.Response,
)
all_embeddings = [data["data"] for data in result.json()["data"]]
ret = []
for sent_tokens, sent_emb in zip(all_tokens, all_embeddings):
token_embs = dict[int, float]()
if sent_tokens[0] == 0:
sent_tokens = sent_tokens[1:]
for token, val in zip(sent_tokens, sent_emb):
token_embs[token] = max(val, token_embs.get(token, 0.0))
ret.append(token_embs)
return ret
# Based on https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L129
def compute_lexical_matching_score(
lw1: dict[int, float], lw2: dict[int, float]
) -> float:
scores = 0.0
for token, weight in lw1.items():
if token in lw2:
scores += weight * lw2[token]
return scores
@pytest.mark.asyncio
async def test_bge_m3_api_server_sparse_embedding(server, pooling_task):
client = server.get_async_client()
if pooling_task != "token_classify":
with pytest.raises(openai.BadRequestError):
await sparse_embeddings(client, sentences_1)
return
embeddings_1 = await sparse_embeddings(client, sentences_1)
embeddings_2 = await sparse_embeddings(client, sentences_2)
lexical_scores_1_0_x_2_0 = compute_lexical_matching_score(
embeddings_1[0], embeddings_2[0]
)
assert lexical_scores_1_0_x_2_0 == pytest.approx(
lexical_score_reference[0], rel=0.01
)
lexical_scores_1_0_x_1_1 = compute_lexical_matching_score(
embeddings_1[0], embeddings_1[1]
)
assert lexical_scores_1_0_x_1_1 == pytest.approx(
lexical_score_reference[1], rel=0.01
)
@pytest.mark.asyncio
async def test_bge_m3_api_server_sparse_embedding_corner_case(server, pooling_task):
if pooling_task != "token_classify":
return
client = server.get_async_client()
embeddings = await sparse_embeddings(client, ["Hi"])
assert len(embeddings) == 1
assert 2673 in embeddings[0]
assert embeddings[0][2673] == pytest.approx(0.26710861921310425, rel=0.01)
# https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L163
def colbert_score(q_reps: torch.Tensor, p_reps: torch.Tensor) -> torch.Tensor:
token_scores = torch.einsum("in,jn->ij", q_reps, p_reps)
scores, _ = token_scores.max(-1)
scores = torch.sum(scores) / q_reps.size(0)
return scores
@pytest.mark.asyncio
async def test_bge_m3_api_server_multi_vector(server, pooling_task):
client = server.get_async_client()
if pooling_task != "token_embed":
with pytest.raises(openai.BadRequestError):
await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
cast_to=httpx.Response,
)
return
result_1 = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
cast_to=httpx.Response,
)
embeddings_1 = [torch.tensor(data["data"]) for data in result_1.json()["data"]]
result_2 = await client.post(
"../pooling",
body={"model": MODEL_NAME, "input": sentences_2, "task": "token_embed"},
cast_to=httpx.Response,
)
embeddings_2 = [torch.tensor(data["data"]) for data in result_2.json()["data"]]
colbert_score_1_0_x_2_0 = colbert_score(embeddings_1[0], embeddings_2[0])
assert colbert_score_1_0_x_2_0 == pytest.approx(
colbert_score_reference[0], rel=0.01
)
colbert_score_1_0_x_2_1 = colbert_score(embeddings_1[0], embeddings_2[1])
assert colbert_score_1_0_x_2_1 == pytest.approx(
colbert_score_reference[1], rel=0.01
)
@@ -1,108 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"model",
[
pytest.param(
"jason9693/Qwen2.5-1.5B-apeach",
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
],
)
@pytest.mark.parametrize("dtype", ["half"] if current_platform.is_rocm() else ["float"])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
# the tolerance value of 1e-2 is selected based on the
# half datatype tests in
# tests/models/language/pooling/test_embedding.py
assert torch.allclose(
hf_output,
vllm_output,
rtol=2e-3 if dtype == "float" else 1e-2,
)
@pytest.mark.core_model
def test_bert_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
score_inputs = (
"What is the capital of France?",
[
"Paris.",
"Paris is the capital and largest city of France.",
"William Shakespeare wrote Hamlet in the early seventeenth century.",
],
)
prompt_batches = [
["short input"],
[
"short input",
"a longer input that exercises mixed sequence lengths",
],
]
with hf_runner(
model, dtype="half", auto_cls=AutoModelForSequenceClassification
) as hf_model:
# HfRunner uses problem_type to preserve the model's
# sbert_ce_default_activation_function=Identity raw logits.
hf_model.config.problem_type = "regression"
hf_outputs = [hf_model.classify(prompts) for prompts in prompt_batches]
text_1, text_2 = score_inputs
text_pairs = [[text_1, document] for document in text_2]
with hf_runner(model, dtype="half", is_cross_encoder=True) as hf_model:
hf_scores = hf_model.predict(text_pairs).tolist()
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
dtype="half",
max_model_len=64,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = [vllm_model.classify(prompts) for prompts in prompt_batches]
vllm_scores = vllm_model.score(*score_inputs)
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
hf_tensor = torch.tensor(hf_batch)
vllm_tensor = torch.tensor(vllm_batch)
assert vllm_tensor.shape == hf_tensor.shape
assert torch.allclose(vllm_tensor, hf_tensor, rtol=1e-2, atol=1e-4)
assert torch.allclose(
torch.tensor(vllm_scores),
torch.tensor(hf_scores),
rtol=1e-2,
atol=1e-4,
)
@@ -1,374 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColBERT late interaction scoring.
Tests are parametrized across multiple ColBERT backbones to ensure the
generic ColBERT support works with different encoder architectures.
"""
from contextlib import contextmanager
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
# -----------------------------------------------------------------------
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
# -----------------------------------------------------------------------
COLBERT_MODELS = {
"bert": {
"model": "answerdotai/answerai-colbert-small-v1",
"colbert_dim": 96,
"max_model_len": 512,
"extra_kwargs": {},
"hf_comparison": {
"weights_file": "model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "BertModel",
},
},
"modernbert": {
"model": "lightonai/GTE-ModernColBERT-v1",
"colbert_dim": 128,
"max_model_len": 299,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTModernBertModel"],
},
},
"hf_comparison": {
"weights_file": "1_Dense/model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "AutoModel",
},
},
"jina": {
"model": "jinaai/jina-colbert-v2",
"colbert_dim": 128,
"max_model_len": 8192,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTJinaRobertaModel"],
},
},
"hf_comparison": {
"weights_file": "model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": True,
"model_cls": "AutoModel",
},
},
"lfm2": {
"model": "LiquidAI/LFM2-ColBERT-350M",
"colbert_dim": 128,
"max_model_len": 511,
"extra_kwargs": {
"hf_overrides": {
"architectures": ["ColBERTLfm2Model"],
},
},
"hf_comparison": {
"weights_file": "1_Dense/model.safetensors",
"weights_key": "linear.weight",
"trust_remote_code": False,
"model_cls": "AutoModel",
},
},
}
TEXTS_1 = [
"What is the capital of France?",
"What is the capital of Germany?",
]
TEXTS_2 = [
"The capital of France is Paris.",
"The capital of Germany is Berlin.",
]
DTYPE = "half"
def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load HF model on the given device with a compatible attention impl."""
from transformers import AutoModel, BertModel
cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel
trust = hf_spec.get("trust_remote_code", False)
# Flash / Triton kernels require GPU tensors; fall back to eager on CPU.
extra = {}
if device.type == "cpu":
extra["attn_implementation"] = "eager"
model = cls.from_pretrained(
model_name,
trust_remote_code=trust,
**extra,
).to(device)
model.eval()
# Transformers 5.0 weight materialization can clear non-persistent
# buffers (e.g. rotary inv_freq) that were registered with
# persistent=False. Re-compute them so the model produces valid output.
for mod in model.modules():
if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
mod.inv_freq = mod._compute_inv_freq(device=device)
return model
def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device):
"""Download and return the ColBERT linear projection weight."""
from safetensors.torch import load_file
from vllm.transformers_utils.repo_utils import hf_api, with_retry
path = with_retry(
lambda: hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]),
f"Error downloading {hf_spec['weights_file']} from {model_name}",
)
weights = load_file(path)
return weights[hf_spec["weights_key"]].to(device)
def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device):
"""Run HF model + projection and return L2-normalised token embeddings."""
import torch.nn.functional as F
embeddings = []
for text in texts:
inputs = tokenizer(text, return_tensors="pt").to(device)
with torch.no_grad():
hidden = model(**inputs).last_hidden_state.float()
projected = F.linear(hidden, linear_weight.float())
normalised = F.normalize(projected, p=2, dim=-1)
embeddings.append(normalised.squeeze(0).cpu())
return embeddings
@contextmanager
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
These live outside any runner context, so without explicit cleanup ROCm
keeps the VRAM resident and the next backend parametrization (or test)
OOMs on startup.
"""
hf_model = _load_hf_model(model_name, hf_spec, device)
linear_weight = _load_projection_weight(model_name, hf_spec, device)
try:
yield hf_model, linear_weight
finally:
del hf_model, linear_weight
cleanup_dist_env_and_memory()
wait_for_memory_to_settle()
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
"""Assert that vLLM and HuggingFace embeddings match."""
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
vllm_emb = torch.as_tensor(vllm_out).float()
assert hf_emb.shape == vllm_emb.shape, (
f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}"
)
torch.testing.assert_close(
vllm_emb,
hf_emb,
rtol=1e-2,
atol=1e-2,
msg=f"Embedding mismatch for text {i}",
)
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class")
def colbert_spec(request):
"""Return the model spec dict for the current parametrization."""
return COLBERT_MODELS[request.param]
@pytest.fixture(scope="class")
def colbert_model_name(colbert_spec):
return colbert_spec["model"]
@pytest.fixture(scope="class")
def colbert_dim(colbert_spec):
return colbert_spec["colbert_dim"]
@pytest.fixture(scope="class")
def colbert_max_model_len(colbert_spec):
return colbert_spec["max_model_len"]
@pytest.fixture(scope="class")
def colbert_extra_kwargs(colbert_spec):
return colbert_spec["extra_kwargs"]
@pytest.fixture(scope="class")
def colbert_model(
vllm_runner,
colbert_model_name,
colbert_max_model_len,
colbert_extra_kwargs,
):
with vllm_runner(
colbert_model_name,
runner="pooling",
dtype=DTYPE,
max_model_len=colbert_max_model_len,
enforce_eager=True,
**colbert_extra_kwargs,
) as vllm_model:
yield vllm_model
class TestColbertSharedEngine:
"""Tests sharing one engine per model.
Class-scoped so the engine is released before `test_colbert_hf_comparison`,
which needs a runner of its own and cannot start while this one holds VRAM.
"""
def test_colbert_token_embed(self, colbert_model, colbert_dim):
"""Test that ColBERT model produces token embeddings."""
outputs = colbert_model.token_embed([TEXTS_1[0]])
assert len(outputs) == 1
emb = torch.as_tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == colbert_dim
assert emb.shape[0] > 1
def test_colbert_late_interaction_1_to_1(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
q_emb = torch.as_tensor(q_outputs[0])
d_emb = torch.as_tensor(d_outputs[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def test_colbert_late_interaction_1_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with 1:N query-documents."""
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
d_outputs = colbert_model.token_embed(TEXTS_2)
q_emb = torch.as_tensor(q_outputs[0])
manual_scores = []
for d_out in d_outputs:
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_late_interaction_N_to_N(self, colbert_model):
"""Test ColBERT late interaction scoring with N:N query-documents."""
q_outputs = colbert_model.token_embed(TEXTS_1)
d_outputs = colbert_model.token_embed(TEXTS_2)
manual_scores = []
for q_out, d_out in zip(q_outputs, d_outputs):
q_emb = torch.as_tensor(q_out)
d_emb = torch.as_tensor(d_out)
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
assert len(vllm_scores) == 2
for i in range(2):
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
def test_colbert_relevance_ordering(self, colbert_model):
"""Test that ColBERT scores relevant documents higher than irrelevant."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"Python is a programming language.",
"Deep learning uses neural networks.",
]
scores = colbert_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
def test_colbert_embed_not_supported(self, colbert_model):
"""Test that ColBERT model does not support the embed task."""
with pytest.raises(ValueError, match="Embedding API is not supported"):
colbert_model.embed([TEXTS_1[0]])
@pytest.mark.parametrize(
("backend", "use_v2"),
[
pytest.param("bert", True, id="bert-v2"),
pytest.param("modernbert", True, id="modernbert-v2"),
pytest.param("jina", True, id="jina-v2"),
pytest.param("lfm2", True, id="lfm2-v2"),
],
)
def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2):
"""Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
from transformers import AutoTokenizer
spec = COLBERT_MODELS[backend]
hf_spec = spec["hf_comparison"]
extra_kwargs = spec["extra_kwargs"]
model_name = spec["model"]
assert isinstance(model_name, str)
assert isinstance(hf_spec, dict)
assert isinstance(extra_kwargs, dict)
test_texts = [TEXTS_1[0], TEXTS_2[0]]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0")
with vllm_runner(
model_name,
runner="pooling",
dtype="float32",
max_model_len=spec["max_model_len"],
enforce_eager=True,
**extra_kwargs,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2
vllm_outputs = vllm_model.token_embed(test_texts)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
hf_tokenizer = AutoTokenizer.from_pretrained(
model_name,
trust_remote_code=hf_spec.get("trust_remote_code", False),
)
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
hf_embeddings = _compute_hf_colbert_embeddings(
hf_model,
hf_tokenizer,
linear_weight,
test_texts,
device,
)
_assert_embeddings_close(vllm_outputs, hf_embeddings)
@@ -1,219 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from vllm import PoolingParams
from vllm.config import PoolerConfig
from ...utils import check_embeddings_close
@pytest.mark.parametrize(
"model",
[
# Be careful of the order of models, decoder-only models should be
# placed before encoder-only models, otherwise `Qwen2.5-0.5B-Instruct`
# case won't pass because gte-Qwen2-1.5B-instruct will cache custom
# model code with bidirectional attention.
# [Decoder-only]
pytest.param(
"BAAI/bge-multilingual-gemma2",
marks=[pytest.mark.core_model, pytest.mark.slow_test],
),
pytest.param(
"intfloat/e5-mistral-7b-instruct",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"ssmits/Qwen2-7B-Instruct-embed-base", marks=[pytest.mark.cpu_model]
),
# [Encoder-only]
pytest.param(
"BAAI/bge-base-en-v1.5",
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
pytest.param("intfloat/multilingual-e5-small"),
# [Cross-Encoder]
pytest.param(
"sentence-transformers/stsb-roberta-base-v2",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model,
) -> None:
vllm_extra_kwargs = {}
if model == "ssmits/Qwen2-7B-Instruct-embed-base":
vllm_extra_kwargs["pooler_config"] = PoolerConfig(
seq_pooling_type="MEAN", use_activation=False
)
max_model_len: int | None = 512
if model in [
"sentence-transformers/all-MiniLM-L12-v2",
"sentence-transformers/stsb-roberta-base-v2",
]:
max_model_len = None
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = hf_model.encode(example_prompts)
with vllm_runner(
model, runner="pooling", max_model_len=max_model_len, **vllm_extra_kwargs
) as vllm_model:
vllm_outputs = vllm_model.embed(example_prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.parametrize(
"model",
[
"BAAI/bge-base-en-v1.5",
"intfloat/multilingual-e5-small",
],
)
@torch.inference_mode()
def test_encoder_only_model_runner_v2_attention(
hf_runner,
vllm_runner,
monkeypatch,
model: str,
) -> None:
prompts = [
"short input",
"a longer input that exercises mixed sequence lengths",
]
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
hf_outputs = []
for prompt in prompts:
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
output = hf_model.model(**hf_model.wrap_device(inputs))
embedding = torch.nn.functional.normalize(
output.last_hidden_state[0, -1].float(), dim=0
)
hf_outputs.append(embedding.cpu().tolist())
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
dtype="float",
max_model_len=64,
max_num_seqs=2,
gpu_memory_utilization=0.25,
pooler_config=PoolerConfig(
task="embed", seq_pooling_type="LAST", use_activation=True
),
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.core_model
def test_encoder_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
model = "sentence-transformers/all-MiniLM-L6-v2"
prompt_batches = [
["short input"],
[
"short input",
"a longer input that exercises mixed sequence lengths",
],
]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = [hf_model.encode(prompts) for prompts in prompt_batches]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
max_model_len=64,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = [vllm_model.embed(prompts) for prompts in prompt_batches]
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_batch,
embeddings_1_lst=vllm_batch,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@pytest.mark.core_model
def test_matryoshka_dimensions_model_runner_v2(
hf_runner, vllm_runner, monkeypatch
) -> None:
model = "Snowflake/snowflake-arctic-embed-m-v1.5"
prompts = ["short input", "a longer input for a different output width"]
dimensions = [None, 256]
with hf_runner(model, is_sentence_transformer=True) as hf_model:
hf_outputs = hf_model.encode(prompts)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
max_model_len=64,
gpu_memory_utilization=0.25,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.embed(
prompts,
pooling_params=[PoolingParams(dimensions=d) for d in dimensions],
)
expected_outputs = []
for output, dimension in zip(hf_outputs, dimensions):
output = torch.as_tensor(output)
if dimension is not None:
output = torch.nn.functional.normalize(output[:dimension], dim=0)
expected_outputs.append(output.tolist())
assert [len(output) for output in vllm_outputs] == [768, 256]
check_embeddings_close(
embeddings_0_lst=expected_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,48 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm import TokensPrompt
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["Qwen/Qwen3-0.6B"],
)
@torch.inference_mode
def test_extract_hidden_states(hf_runner, vllm_runner, model: str):
n_prompt_tokens = [55, 56, 57]
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
with vllm_runner(
model,
max_model_len=128,
enforce_eager=True,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
enable_prefix_caching=True,
) as vllm_model:
pooling_outputs = vllm_model.llm.encode(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
pooling_task="token_embed",
)
for n, output in zip(n_prompt_tokens, pooling_outputs):
assert len(output.prompt_token_ids) == n
assert len(output.outputs.data) == n
assert output.num_cached_tokens == 0
# test enable_prefix_caching plus all pooling
# we need to skip reading cache at this request by
# request.skip_reading_prefix_cache
pooling_outputs = vllm_model.llm.encode(
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
pooling_task="token_embed",
)
for n, output in zip(n_prompt_tokens, pooling_outputs):
assert len(output.prompt_token_ids) == n
assert len(output.outputs.data) == n
assert output.num_cached_tokens == 0
@@ -1,47 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
@pytest.mark.parametrize(
"model",
["nie3e/sentiment-polish-gpt2-small"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for head_dtype_str in ["float32", "model"]:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
hf_overrides={"head_dtype": head_dtype_str},
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
model_dtype = model_config.dtype
head_dtype = model_config.head_dtype
if head_dtype_str == "float32":
assert head_dtype == torch.float32
elif head_dtype_str == "model":
assert head_dtype == model_dtype
vllm_outputs = vllm_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Backbone validation for Jina Embeddings V5.
The V5 family ships two backbones under one `architectures` entry: `-small` is
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
ships a separate `configuration_*.py` per repository, so the only signal
distinguishing them is `is_decoder`, which the encoder variant sets to False.
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
"""
from types import SimpleNamespace
from typing import cast
import pytest
from transformers import PreTrainedConfig
from vllm.config import ModelConfig
from vllm.model_executor.models.config import (
MODELS_CONFIG_MAP,
JinaEmbeddingsV5ModelConfig,
)
def _model_config(hf_config: PreTrainedConfig) -> ModelConfig:
"""Minimal stand-in for ModelConfig; only hf_config is read."""
return cast(ModelConfig, SimpleNamespace(hf_config=hf_config))
@pytest.mark.cpu_test
def test_registered_for_the_architecture():
"""The handler only runs if it is wired to the architecture name."""
assert MODELS_CONFIG_MAP["JinaEmbeddingsV5Model"] is JinaEmbeddingsV5ModelConfig
@pytest.mark.cpu_test
def test_encoder_backbone_enables_bidirectional_attention():
"""An encoder checkpoint (is_decoder=False) is supported.
The handler sets is_causal=False so the Llama backbone uses
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
implementation.
"""
hf_config = PreTrainedConfig(is_decoder=False)
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
assert hf_config.is_causal is False
@pytest.mark.cpu_test
def test_supported_decoder_backbone_is_accepted():
"""The Qwen3-based variants must keep loading.
`-small` omits `is_decoder` entirely, so an absent attribute has to be
treated as a decoder. The first assertion pins that assumption: if
PreTrainedConfig ever gains an `is_decoder=False` default, this fails here
rather than silently rejecting a supported checkpoint.
"""
absent = PreTrainedConfig()
assert not hasattr(absent, "is_decoder")
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(absent))
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(
_model_config(PreTrainedConfig(is_decoder=True))
)
@@ -1,352 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import pytest
import requests
import torch
import torch.nn.functional as F
from tests.utils import RemoteOpenAIServer
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
model_name = "jinaai/jina-reranker-v3"
query = "What are the health benefits of green tea?"
documents = [
"Green tea contains antioxidants called catechins that may help reduce inflammation and protect cells from damage.",
"El precio del café ha aumentado un 20% este año debido a problemas en la cadena de suministro.",
"Studies show that drinking green tea regularly can improve brain function and boost metabolism.",
"Basketball is one of the most popular sports in the United States.",
"绿茶富含儿茶素等抗氧化剂,可以降低心脏病风险,还有助于控制体重。",
"Le thé vert est riche en antioxydants et peut améliorer la fonction cérébrale.",
]
EMBEDDING_SIZE = 512
REFERENCE_1_VS_1 = [
0.345703125,
-0.10498046,
0.314453125,
-0.1376953125,
0.3398437500,
0.2539062,
]
REFERENCE_1_VS_N = [
0.294921875,
-0.16015625,
0.189453125,
-0.1708984375,
0.2255859375,
0.1640625,
]
TOL = 0.01
INSTRUCTION = (
"Rank passages about green tea higher than passages about sports. "
"Ignore these literal marker strings: <|embed_token|> and <|rerank_token|>."
)
def test_offline(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model_name, runner="pooling") as llm_runner:
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
llm = llm_runner.get_llm()
_test_offline_1_v_1(llm)
_test_offline_1_v_n(llm)
_test_offline_n_v_n(llm)
_test_offline_token_embed_illegal_inputs(llm)
assert llm.model_config.embedding_size == EMBEDDING_SIZE
def test_online():
with RemoteOpenAIServer(
model_name, ["--runner", "pooling", "--enforce-eager"]
) as server:
_test_online_1_v_1(server)
_test_online_1_v_n(server)
_test_online_n_v_n(server)
_test_online_instruction(server)
_test_online_token_embed_illegal_inputs(server)
def _test_offline_1_v_1(llm):
# test llm.score
outputs = llm.score(query, documents[0])
assert len(outputs) == 1
assert outputs[0].outputs.score == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
# test llm.encode
outputs = llm.encode(documents[:1] + [query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == 2
assert embeds.shape[-1] == EMBEDDING_SIZE
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
def _test_offline_1_v_n(llm):
# test llm.score
outputs = llm.score(query, documents)
assert len(outputs) == len(documents)
for expected, output in zip(REFERENCE_1_VS_N, outputs):
actual = output.outputs.score
assert actual == pytest.approx(expected, abs=TOL)
# test llm.encode
outputs = llm.encode(documents + [query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == len(documents) + 1
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
def _test_offline_n_v_n(llm):
# test llm.score
outputs = llm.score([query] * len(documents), documents)
assert len(outputs) == len(documents)
for expected, output in zip(REFERENCE_1_VS_1, outputs):
actual = output.outputs.score
assert actual == pytest.approx(expected, abs=TOL)
# test llm.encode
for doc, expected in zip(documents, REFERENCE_1_VS_1):
outputs = llm.encode([doc, query], pooling_task="token_embed")
embeds = outputs[0].outputs.data.float()
assert embeds.shape[0] == 2
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(expected, abs=TOL)
def _test_offline_token_embed_illegal_inputs(llm):
with pytest.raises(
ValueError, match="The JinaForRanking model requires at least 2 inputs."
):
llm.encode([query], pooling_task="token_embed")
with pytest.raises(
ValueError, match="The JinaForRanking model only supports text as input."
):
llm.encode([1, 2, 3], pooling_task="token_embed")
def _get_score_response(server, query, document, **extra_body):
payload = {
"model": model_name,
"queries": query,
"documents": document,
}
payload.update(extra_body)
score_response = requests.post(
server.url_for("score"),
json=payload,
)
score_response.raise_for_status()
return ScoreResponse.model_validate(score_response.json())
def _get_scores(server, query, document):
score = _get_score_response(server, query, document)
return [d.score for d in score.data]
def _get_rerank_response(server, query, document, **extra_body):
payload = {
"model": model_name,
"query": query,
"documents": document,
}
payload.update(extra_body)
rerank_response = requests.post(
server.url_for("rerank"),
json=payload,
)
rerank_response.raise_for_status()
return RerankResponse.model_validate(rerank_response.json())
def _get_embeds(server, prompts: list[str]):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": prompts,
"encoding_format": "float",
},
)
response.raise_for_status()
poolings = PoolingResponse.model_validate(response.json())
return torch.as_tensor([d.data for d in poolings.data][0]).float()
def _test_online_1_v_1(server):
# test scoring api
scores = _get_scores(server, query, documents[0])
assert len(scores) == 1
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
# test pooling api
embeds = _get_embeds(server, [documents[0], query])
assert embeds.shape[0] == 2
assert embeds.shape[-1] == EMBEDDING_SIZE
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
def _test_online_1_v_n(server):
# test scoring api
scores = _get_scores(server, query, documents)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
# test pooling api
embeds = _get_embeds(server, documents + [query])
assert embeds.shape[0] == len(documents) + 1
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_N, scores):
assert actual == pytest.approx(expected, abs=TOL)
def _test_online_n_v_n(server):
# test scoring api
scores = _get_scores(server, [query] * len(documents), documents)
assert len(scores) == len(documents)
for expected, actual in zip(REFERENCE_1_VS_1, scores):
assert actual == pytest.approx(expected, abs=TOL)
# test pooling api
for doc, expected in zip(documents, REFERENCE_1_VS_1):
embeds = _get_embeds(server, [doc, query])
assert embeds.shape[0] == 2
doc_embeds = embeds[:-1]
query_embeds = embeds[-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
assert len(scores) == 1
assert scores[0] == pytest.approx(expected, abs=TOL)
def _test_online_instruction(server):
docs = documents[:2]
default_score = _get_score_response(server, query, docs)
instruction_score = _get_score_response(
server,
query,
docs,
instruction=INSTRUCTION,
)
kwargs_score = _get_score_response(
server,
query,
docs,
chat_template_kwargs={"instruction": INSTRUCTION},
)
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
assert kwargs_score.usage.prompt_tokens == instruction_score.usage.prompt_tokens
assert len(instruction_score.data) == len(default_score.data)
assert [d.score for d in kwargs_score.data] == pytest.approx(
[d.score for d in instruction_score.data], abs=TOL
)
default_rerank = _get_rerank_response(server, query, docs)
instruction_rerank = _get_rerank_response(
server,
query,
docs,
instruction=INSTRUCTION,
)
kwargs_rerank = _get_rerank_response(
server,
query,
docs,
chat_template_kwargs={"instruction": INSTRUCTION},
)
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
assert kwargs_rerank.usage.prompt_tokens == instruction_rerank.usage.prompt_tokens
assert len(instruction_rerank.results) == len(default_rerank.results)
assert [r.relevance_score for r in kwargs_rerank.results] == pytest.approx(
[r.relevance_score for r in instruction_rerank.results], abs=TOL
)
def _test_online_token_embed_illegal_inputs(server):
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": [query],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking model requires at least 2 inputs."
)
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"input": [1, 2, 3],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking model only supports text as input."
)
response = requests.post(
server.url_for("pooling"),
json={
"model": model_name,
"task": "token_embed",
"messages": [
{
"role": "user",
"content": "The cat sat on the mat.",
}
],
"encoding_format": "float",
},
)
assert response.json()["error"]["message"].startswith(
"The JinaForRanking does not support chat Request."
)
@@ -1,227 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
import json
import os
from dataclasses import dataclass
import pytest
import requests
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
ExpectedPromptTokens = int | tuple[int, ...]
long_query = "What is the capital of France?" * 20
long_doc = "The capital of France is Paris. " * 20
@dataclass
class TestConfig:
model: str
args: list[str]
without_truncated_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_query_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_doc_prompt_tokens: ExpectedPromptTokens
with_max_tokens_per_query_and_doc_prompt_tokens: ExpectedPromptTokens
RERANK_CONFIGS = [
# 1. cross-encoder
TestConfig(
model="jinaai/jina-reranker-v2-base-multilingual",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--trust-remote-code",
],
without_truncated_prompt_tokens=284,
with_max_tokens_per_query_prompt_tokens=154,
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 2. cross-encoder + score template
TestConfig(
model="Qwen/Qwen3-Reranker-0.6B",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--hf-overrides",
json.dumps(
{
"architectures": ["Qwen3ForSequenceClassification"],
"classifier_from_token": ["no", "yes"],
"is_original_qwen3_reranker": True,
}
),
"--chat-template",
os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
],
without_truncated_prompt_tokens=352,
with_max_tokens_per_query_prompt_tokens=223,
with_max_tokens_per_doc_prompt_tokens=221,
with_max_tokens_per_query_and_doc_prompt_tokens=92,
),
# 3. bi-encoder
TestConfig(
model="intfloat/multilingual-e5-small",
args=[
"--enforce-eager",
"--max-model-len",
"512",
"--trust-remote-code",
],
# This model has produced both prompt-token totals in CI/local cache;
# keep truncation checks exact while tolerating the boundary delta.
without_truncated_prompt_tokens=(284, 285),
with_max_tokens_per_query_prompt_tokens=(154, 155),
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 4. late-interaction
TestConfig(
model="answerdotai/answerai-colbert-small-v1",
args=[
"--enforce-eager",
"--max-model-len",
"512",
"--trust-remote-code",
],
without_truncated_prompt_tokens=284,
with_max_tokens_per_query_prompt_tokens=154,
with_max_tokens_per_doc_prompt_tokens=154,
with_max_tokens_per_query_and_doc_prompt_tokens=24,
),
# 5. jinaai/jina-reranker-v3
TestConfig(
model="jinaai/jina-reranker-v3",
args=[
"--enforce-eager",
"--max-model-len",
"1024",
"--trust-remote-code",
],
without_truncated_prompt_tokens=567,
with_max_tokens_per_query_prompt_tokens=308,
with_max_tokens_per_doc_prompt_tokens=436,
with_max_tokens_per_query_and_doc_prompt_tokens=177,
),
]
def assert_prompt_tokens(actual: int, expected: ExpectedPromptTokens) -> None:
if isinstance(expected, int):
assert actual == expected
else:
assert actual in expected
@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
def server(request):
config: TestConfig = request.param
env_dict = {"VLLM_LOGGING_LEVEL": "WARNING"}
with RemoteOpenAIServer(
config.model, config.args, env_dict=env_dict
) as remote_server:
yield config, remote_server
def test_without_truncated(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={"model": config.model, "query": long_query, "documents": [long_doc]},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.without_truncated_prompt_tokens,
)
def test_max_tokens_per_query(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_query": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_query_prompt_tokens,
)
def test_max_tokens_per_doc(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_doc": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_doc_prompt_tokens,
)
def test_max_tokens_per_query_and_doc(server):
"""Test that max_tokens_per_doc truncates documents correctly."""
config, remote_server = server
response = requests.post(
remote_server.url_for("rerank"),
json={
"model": config.model,
"query": long_query,
"documents": [long_doc],
"max_tokens_per_query": 10,
"max_tokens_per_doc": 10,
},
)
response.raise_for_status()
rerank = RerankResponse.model_validate(response.json())
assert rerank.id is not None
assert rerank.results is not None
assert len(rerank.results) == 1
assert_prompt_tokens(
rerank.usage.prompt_tokens,
config.with_max_tokens_per_query_and_doc_prompt_tokens,
)
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.config.pooler import PoolerConfig
def test_idefics_multimodal(
vllm_runner,
) -> None:
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
with vllm_runner(
model_name="HuggingFaceM4/Idefics3-8B-Llama3",
runner="pooling",
convert="classify",
load_format="dummy",
max_model_len=512,
enforce_eager=True,
tensor_parallel_size=1,
disable_log_stats=True,
dtype="bfloat16",
) as vllm_model:
llm = vllm_model.get_llm()
outputs = llm.classify(prompts)
for output in outputs:
assert len(output.outputs.probs) == 2
def update_config(config):
text_config = config.get_text_config()
text_config.update(
{
"architectures": ["Gemma3ForSequenceClassification"],
"classifier_from_token": ["A", "B", "C", "D", "E"],
"method": "no_post_processing",
"id2label": {
"A": "Chair",
"B": "Couch",
"C": "Table",
"D": "Bed",
"E": "Cupboard",
},
}
)
return config
def test_gemma_multimodal(
vllm_runner,
) -> None:
messages = [
{
"role": "system",
"content": """
You are a helpful assistant. You will be given a product description
which may also include an image. Classify the following product into
one of the categories:
A = chair
B = couch
C = table
D = bed
E = cupboard
You'll answer with exactly one letter (A, B, C, D, or E).""",
},
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/red_chair.jpg"
},
},
{"type": "text", "text": "A fine 19th century piece of furniture."},
],
},
]
with vllm_runner(
model_name="google/gemma-3-4b-it",
runner="pooling",
convert="classify",
load_format="auto",
hf_overrides=update_config,
pooler_config=PoolerConfig(seq_pooling_type="LAST"),
max_model_len=512,
enforce_eager=True,
tensor_parallel_size=1,
disable_log_stats=True,
dtype="bfloat16",
) as vllm_model:
llm = vllm_model.get_llm()
prompts = llm._preprocess_chat([messages])
result = llm.classify(prompts)
assert result[0].outputs.probs[0] > 0.95
assert all(c < 0.05 for c in result[0].outputs.probs[1:])
@@ -1,44 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from tests.models.utils import check_embeddings_close
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["BAAI/bge-m3"],
)
@pytest.mark.parametrize("dtype", ["float"])
@torch.inference_mode
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
with vllm_runner(
model,
runner="pooling",
pooler_config=PoolerConfig(task="token_embed"),
max_model_len=None,
dtype=dtype,
) as vllm_model:
vllm_outputs = vllm_model.token_embed(example_prompts)
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
embedding = output.last_hidden_state[0].float()
hf_outputs.append(embedding.cpu())
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
check_embeddings_close(
embeddings_0_lst=hf_output,
embeddings_1_lst=vllm_output,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,34 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForSequenceClassification
@pytest.mark.parametrize(
"model",
["Rami/multi-label-class-classification-on-github-issues"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
) as hf_model:
hf_outputs = hf_model.classify(example_prompts)
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output)
vllm_output = torch.tensor(vllm_output)
assert torch.allclose(
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
)
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: SIM117
from functools import partial
import pytest
from ...utils import EmbedModelInfo
MODELS = [
EmbedModelInfo(
"nomic-ai/nomic-embed-text-v1",
# Fixme:
# Update nomic-embed code to support the latest
# HF version and remove revision set.
revision="720244025c1a7e15661a174c63cce63c8218e52b",
),
# EmbedModelInfo("nomic-ai/nomic-embed-text-v1.5"),
# EmbedModelInfo("nomic-ai/CodeRankEmbed"),
EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe"),
# EmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long"),
]
rope_theta = 1000
factor = 4.0
original_max_position_embeddings = 2048
max_model_len = int(original_max_position_embeddings * factor)
@pytest.fixture(scope="module", autouse=True)
def predownload_models():
"""Warm the HF cache so engine startup never hits transient hub errors."""
from vllm.transformers_utils.repo_utils import hf_api, with_retry
for model_info in MODELS:
with_retry(
partial(
hf_api().snapshot_download,
model_info.name,
revision=model_info.revision,
),
f"Error downloading {model_info.name}",
)
@pytest.mark.parametrize("model_info", MODELS)
def test_default(model_info, vllm_runner):
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=None,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
# For nomic-embed-text-v2-moe the length is set to 512
# by sentence_bert_config.json.
assert model_config.max_model_len == 512
if model_info.name == "nomic-ai/nomic-embed-text-v1":
assert model_config.max_model_len == 8192
@pytest.mark.parametrize("model_info", MODELS)
def test_set_max_model_len_legal(model_info, vllm_runner):
# set max_model_len <= 512
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=256,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 256
# For nomic-embed-text-v2-moe the length is set to 512
# by sentence_bert_config.json.
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
with pytest.raises(ValueError):
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=1024,
):
pass
return
# set 512 < max_model_len <= 2048
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=1024,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 1024
# set max_model_len > 2048
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=4096,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.max_model_len == 4096
@pytest.mark.parametrize("model_info", MODELS)
def test_use_rope_scaling_legal(model_info, vllm_runner):
hf_overrides = {
"rope_parameters": {
"rope_theta": rope_theta,
"rope_type": "yarn",
"factor": factor,
"original_max_position_embeddings": original_max_position_embeddings,
},
"max_model_len": max_model_len,
}
with vllm_runner(
model_info.name,
revision=model_info.revision,
runner="pooling",
max_model_len=None,
hf_overrides=hf_overrides,
):
pass
@@ -1,167 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
from tests.models.utils import softmax
from vllm.config import PoolerConfig
@pytest.mark.parametrize(
"model",
["jason9693/Qwen2.5-1.5B-apeach", "papluca/xlm-roberta-base-language-detection"],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_classify_models_using_activation(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_activation_out = vllm_model.classify(example_prompts)
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_activation_out = vllm_model.classify(example_prompts)
for wo_activation, w_activation in zip(wo_activation_out, w_activation_out):
wo_activation = torch.tensor(wo_activation)
w_activation = torch.tensor(w_activation)
assert not torch.allclose(wo_activation, w_activation, atol=1e-2), (
"pooler_config is not working"
)
assert torch.allclose(
softmax(wo_activation), w_activation, 1e-3 if dtype == "float" else 1e-2
)
@pytest.mark.parametrize(
"model",
[
"intfloat/multilingual-e5-small",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_embed_models_using_normalize(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_normalize = torch.tensor(vllm_model.embed(example_prompts))
assert not torch.allclose(wo_normalize, w_normalize, atol=1e-2), (
"pooler_config normalize is not working"
)
assert torch.allclose(
F.normalize(wo_normalize, p=2, dim=-1), w_normalize, atol=1e-2
), "w_normal should be close to normal(wo_normal)."
@pytest.mark.parametrize(
"model",
[
"internlm/internlm2-1_8b-reward",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_reward_models_using_activation(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False),
) as vllm_model:
wo_activation = vllm_model.token_classify(example_prompts)
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True),
) as vllm_model:
w_activation = vllm_model.token_classify(example_prompts)
for wo, w in zip(wo_activation, w_activation):
wo = torch.tensor(wo)
w = torch.tensor(w)
assert not torch.allclose(wo, w, atol=1e-2), (
"pooler_config activation is not working"
)
assert torch.allclose(softmax(wo), w, atol=1e-2), (
"w_activation should be close to activation(wo_activation)."
)
@pytest.mark.parametrize(
"model",
[
"intfloat/multilingual-e5-small",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_multi_vector_retrieval_models_using_normalize(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=False, task="token_embed"),
) as vllm_model:
wo_normalize = vllm_model.token_embed(example_prompts)
with vllm_runner(
model,
max_model_len=512,
dtype=dtype,
pooler_config=PoolerConfig(use_activation=True, task="token_embed"),
) as vllm_model:
w_normalize = vllm_model.token_embed(example_prompts)
for wo, w in zip(wo_normalize, w_normalize):
assert not torch.allclose(wo, w, atol=1e-2), (
"pooler_config normalize is not working"
)
assert torch.allclose(F.normalize(wo, p=2, dim=-1), w, atol=1e-2), (
"w_normal should be close to normal(wo_normal)."
)
@@ -1,179 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import TYPE_CHECKING
import pytest
import torch
import torch.nn.functional as F
from transformers import AutoModel
from vllm.platforms import current_platform
from vllm.utils.mem_constants import MiB_bytes
from ....conftest import HfRunner
from ....utils import VLLM_PATH
from ...registry import HF_EXAMPLE_MODELS
if TYPE_CHECKING:
from _typeshed import StrPath
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
assert FIXTURES_PATH.exists()
FIXTURE_REWARD_RESULT = {
"Qwen/Qwen2.5-Math-PRM-7B": FIXTURES_PATH / "qwen2_5_math_prm_reward_step.json",
}
@pytest.fixture
def math_step_prompts():
# ruff: noqa: E501
data = {
"system": "Please reason step by step, and put your final answer within \\boxed{}. ",
"query": "Sue lives in a fun neighborhood. One weekend, the neighbors decided to play a prank on Sue. On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard. On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and put these newly painted white flamingos back out on Sue's front yard. Then, on Sunday morning, they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more pink plastic flamingos were out than white plastic flamingos?",
"response": [
"To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink plastic flamingos.",
"On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos and 6 white flamingos.",
"On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.",
"To find the difference, subtract the number of white flamingos from the number of pink flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out than white plastic flamingos. The answer is (\\boxed{30}).",
],
}
answer = "<extra_0>".join(data["response"]) + "<extra_0>"
prompt = f"<im_start>system\n{data['system']}<im_end>\n<im_start>user\n{data['query']}<im_end>\n<im_start>assistant\n{answer}<im_end><|endoftext|>"
return [prompt]
def step_reward_patch_hf_model(hf_model: HfRunner):
# Patch the hf_runner to use the step reward function
def make_step_rewards(
logits: torch.Tensor, token_masks: torch.Tensor
) -> list[list[float]]:
probabilities = F.softmax(logits, dim=-1)
probabilities = probabilities * token_masks.unsqueeze(-1)
all_scores_res: list[list[float]] = []
for i in range(probabilities.size(0)):
sample = probabilities[i] # seq_len, num_labels
positive_probs = sample[sample != 0].view(-1, 2)
non_zero_elements_list = positive_probs.cpu().tolist()
all_scores_res.append(non_zero_elements_list)
return all_scores_res
def reward(prompts: list[str]) -> list[list[float]]:
input_ids = hf_model.tokenizer(prompts, return_tensors="pt").input_ids
input_ids = hf_model.wrap_device(input_ids)
outputs = hf_model.model(input_ids=input_ids)
step_sep_id = hf_model.tokenizer.encode("<extra_0>")[0]
token_masks = input_ids == step_sep_id
return make_step_rewards(outputs[0], token_masks)
hf_model.reward = reward # type: ignore[attr-defined]
return hf_model
def dump_reward_outputs(outputs: list[list[float]], filename: "StrPath"):
with open(filename, "w", encoding="utf-8") as f:
json.dump(outputs, f)
def load_reward_outputs(filename: "StrPath") -> list[list[float]]:
with open(filename, encoding="utf-8") as f:
return json.load(f)
@pytest.mark.parametrize(
"model",
[
pytest.param(
"Qwen/Qwen2.5-Math-PRM-7B",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_prm_models(
hf_runner,
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_transformers_version(on_fail="skip")
if current_platform.is_cpu():
pytest.skip("CPU only supports V1")
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_outputs = vllm_model.token_classify(math_step_prompts)
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_model = step_reward_patch_hf_model(hf_model)
hf_outputs = hf_model.reward(math_step_prompts)
dump_reward_outputs(
hf_outputs,
FIXTURE_REWARD_RESULT[model],
)
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = torch.tensor(hf_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(hf_output, vllm_output, 1.5e-2)
@pytest.mark.parametrize(
"model",
[
pytest.param(
"Qwen/Qwen2.5-Math-PRM-7B",
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_prm_models_with_golden_outputs(
vllm_runner,
math_step_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
if not FIXTURE_REWARD_RESULT.get(model):
pytest.skip(f"No available golden outputs for {model}.")
use_v2_model_runner = not current_platform.is_cpu()
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
with vllm_runner(
model,
max_model_len=1024,
dtype=dtype,
kv_cache_memory_bytes=64 * MiB_bytes,
) as vllm_model:
assert (
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
vllm_outputs = vllm_model.token_classify(math_step_prompts)
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
# check logits difference
for golden_output, vllm_output in zip(golden_outputs, vllm_outputs):
golden_output = torch.tensor(golden_output).float()
vllm_output = torch.tensor(vllm_output).float()
assert torch.allclose(golden_output, vllm_output, 1.5e-2)
@@ -1,281 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import numpy as np
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.models.bert import (
BertMLMHead,
SPLADESparsePooler,
)
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
from vllm.v1.worker.gpu.states import RequestState
# ---------------------------------------------------------------------
# Functional test: SPLADE formula correctness (no HF download needed)
# ---------------------------------------------------------------------
@pytest.mark.parametrize("B,T,H,V", [(2, 3, 5, 7)])
@torch.inference_mode
def test_splade_pooler_matches_reference_formula(B, T, H, V):
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
log1p(relu(logits)) -> max over sequence length (after masking)."""
torch.manual_seed(0)
# Prepare [B] sequences of shape [T, H]
hs_list = [torch.randn(T, H) for _ in range(B)]
hs_tenser = torch.cat(hs_list)
# Simulate PoolingMetadata (only required fields)
prompt_lens = [T, T - 1]
prompt_lens_tenser = torch.tensor(prompt_lens, dtype=torch.int32)
token_ids = torch.tensor(
[
[101, 5, 102], # Batch 0: [CLS], token, [SEP]
[101, 6, 6], # Batch 1: [CLS], token, token (last token ignored)
],
dtype=torch.long,
)
meta = PoolingMetadata(
prompt_lens=prompt_lens_tenser,
prompt_token_ids=token_ids,
prompt_token_ids_cpu=token_ids,
pooling_params=[PoolingParams(task="embed")] * B,
pooling_states=[PoolingStates() for _ in range(B)],
)
# MLM head (prefer BertMLMHead, fallback to Linear if unavailable)
try:
mlm_head = BertMLMHead(hidden_size=H, vocab_size=V, layer_norm_eps=1e-12)
except Exception:
mlm_head = nn.Linear(H, V, bias=True)
# Forward pass through SPLADE pooler
pooler = SPLADESparsePooler(mlm_head=mlm_head, pooling="max", remove_cls_sep=True)
pooled = pooler(hidden_states=hs_tenser, pooling_metadata=meta) # list of [V]
# Basic output checks
assert isinstance(pooled, torch.Tensor) and len(pooled) == B
for vec in pooled:
assert vec.shape == (V,)
assert torch.isfinite(vec).all()
assert (vec >= 0).all(), "SPLADE outputs must be non-negative."
# Reference implementation for comparison
def ref_one(hs: torch.Tensor, L: int, tid_row: torch.Tensor) -> torch.Tensor:
keep = torch.ones(L, dtype=torch.bool)
if L > 0 and tid_row[0].item() == 101: # remove CLS
keep[0] = False
if L > 0 and tid_row[L - 1].item() == 102: # remove SEP
keep[L - 1] = False
valid = hs[:L][keep[:L]]
if valid.numel() == 0:
return torch.zeros(V, dtype=torch.float32)
logits = mlm_head(valid) # [L', V]
scores = torch.log1p(torch.relu(logits)) # [L', V]
return scores.max(dim=0).values.to(torch.float32)
torch.testing.assert_close(
pooled[0],
ref_one(hs_list[0], prompt_lens[0], token_ids[0]),
rtol=1e-4,
atol=1e-4,
)
torch.testing.assert_close(
pooled[1],
ref_one(hs_list[1], prompt_lens[1], token_ids[1]),
rtol=1e-4,
atol=1e-4,
)
def test_pooling_runner_gathers_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
runner.pooling_params = {1: pooling_params, 3: pooling_params}
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
runner.prompt_token_ids = {
1: torch.tensor([101, 102]),
3: torch.tensor([101, 11, 102]),
}
input_batch = MagicMock(spec=InputBatch)
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
input_batch.num_reqs = 2
req_states = MagicMock(spec=RequestState)
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
metadata = runner._get_pooling_metadata(
input_batch, req_states, torch.device("cpu")
)
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
assert metadata.prompt_token_ids_cpu is not None
assert metadata.prompt_token_ids is None
assert metadata.prompt_token_ids_cpu.is_pinned() is False
torch.testing.assert_close(
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
)
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
def test_pooling_runner_stores_only_required_token_ids() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.model = MagicMock()
runner.supported_tasks = frozenset({"embed"})
runner.pooling_params = {}
runner.pooling_states = {}
runner.prompt_token_ids = {}
runner.late_interaction_runner = MagicMock()
runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
runner.add_request(
"req-2",
2,
PoolingParams(task="embed", requires_token_ids=True),
[101, 11, 102],
)
assert 1 not in runner.prompt_token_ids
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
runner = PoolingRunner.__new__(PoolingRunner)
runner.late_interaction_runner = LateInteractionRunner()
query_key = "query-abort"
late_interaction_runner = runner.late_interaction_runner
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
late_interaction_runner._query_uses[query_key] = 1
late_interaction_runner._doc_query_keys["doc-req"] = query_key
runner.on_requests_finished({"doc-req"})
assert not late_interaction_runner._query_cache
assert not late_interaction_runner._query_uses
assert not late_interaction_runner._doc_query_keys
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
# Cached query embeddings are only invalidated by weight reloads, which
# reach the runner via reset_encoder_cache. Resetting the multi-modal
# cache is unrelated and must leave them intact.
runner = GPUModelRunner.__new__(GPUModelRunner)
runner.encoder_cache = MagicMock()
runner.pooling_runner = MagicMock()
runner.reset_mm_cache()
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_not_called()
runner.reset_encoder_cache()
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
runner.pooling_runner.clear.assert_called_once_with()
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "plugin"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "plugin"
with (
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
pytest.raises(ValueError, match="selects 'plugin'"),
):
PoolingRunner(model, vllm_config)
def test_pooling_runner_supports_encoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_supports_encoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_embed"}
def test_pooling_runner_supports_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"token_classify"}
def test_pooling_runner_keeps_decoder_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed", "token_classify"}
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "decoder"
vllm_config.model_config.get_pooling_task.return_value = "embed"
runner = PoolingRunner(model, vllm_config)
assert runner.supported_tasks == {"embed", "token_embed"}
def test_pooling_runner_supports_embed_token_classification() -> None:
model = MagicMock()
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_seqs = 2
vllm_config.model_config.attn_type = "encoder_only"
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
assert PoolingRunner(model, vllm_config).supported_tasks == {
"embed",
"embed&token_classify",
}
@@ -1,280 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.models.utils import softmax
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@pytest.fixture(autouse=True)
def seed_everything():
"""Seed all random number generators for reproducibility."""
seed = 0
set_random_seed(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
yield
@pytest.mark.parametrize(
"model",
[
"boltuix/NeuroBERT-NER",
],
)
# The float32 is required for this tiny model to pass the test.
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.core_model
@torch.inference_mode
def test_bert_model_runner_v2(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
dtype: str,
) -> None:
prompt_batches = [[example_prompts[0]], example_prompts]
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
vllm_output_batches = [
vllm_model.token_classify(prompts) for prompts in prompt_batches
]
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
hf_output_batches = [[hf_outputs[0]], hf_outputs]
# check logits difference
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_modernbert_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# NOTE: https://github.com/vllm-project/vllm/pull/32403
# `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
# model, which can cause numerical precision variance and edge cases.
# We use @flaky(reruns=3) to mitigate intermittent failures.
print(
f"\n[NOTE] Testing {model} (randomly initialized weights) - "
"flaky tolerance enabled due to numerical precision variance."
)
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_xlm_roberta_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
PRIVACY_FILTER_PROMPTS = [
"My name is Harry Potter.",
"Email me at harry.potter@hogwarts.edu.",
"Call me on +44 20 7946 0958 tomorrow.",
"My account number is 12345678 and the API key is sk-live-abc123def456.",
"I live at 4 Privet Drive, Little Whinging.",
"Visit https://example.com/profile/harry for more info.",
"We met on 12 January 2024.",
]
@pytest.mark.skipif(
current_platform.is_rocm(),
reason="Workspace growth allocation issue on ROCm. See https://github.com/vllm-project/vllm/issues/48510",
)
@pytest.mark.parametrize("model", ["openai/privacy-filter"])
@pytest.mark.parametrize("dtype", ["bfloat16"])
@torch.inference_mode
def test_openai_privacy_filter(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_transformers_version(on_fail="skip")
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForTokenClassification,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in PRIVACY_FILTER_PROMPTS:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2)
@pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"])
@pytest.mark.parametrize("dtype", ["float"])
@pytest.mark.flaky(reruns=3)
@torch.inference_mode
def test_auto_conversion(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForTokenClassification
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
# check logits difference
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
@pytest.mark.parametrize(
"model",
[
# Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an
# NSP head (`cls.seq_relationship.*`) and a decoder tied to the input
# embeddings (no explicit decoder weight in the checkpoint).
"google-bert/bert-base-uncased",
],
)
@pytest.mark.parametrize("dtype", ["float"])
@torch.inference_mode
def test_bert_for_masked_lm(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
) -> None:
# BertForMaskedLM exposes its MLM head as a token-level pooling task; the
# head applies softmax over the vocabulary, so each output row is a
# distribution (matching HF's softmax(logits) below).
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
vllm_outputs = vllm_model.token_classify(example_prompts)
# Run hf_runner reference with "highest" fp32 precision to match
# default behvior of vLLM. This is needed on ROCm since the
# pooling tests set matmul precision to "high" in conftest.py
prev_matmul_precision = torch.get_float32_matmul_precision()
torch.set_float32_matmul_precision("highest")
with hf_runner(
model,
dtype=dtype,
auto_cls=AutoModelForMaskedLM,
) as hf_model:
tokenizer = hf_model.tokenizer
hf_outputs = []
for prompt in example_prompts:
inputs = tokenizer([prompt], return_tensors="pt")
inputs = hf_model.wrap_device(inputs)
output = hf_model.model(**inputs)
hf_outputs.append(softmax(output.logits[0]))
torch.set_float32_matmul_precision(prev_matmul_precision)
# Compare the per-token vocabulary distributions position by position.
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
hf_output = hf_output.detach().clone().cpu().float()
vllm_output = vllm_output.detach().clone().cpu().float()
assert hf_output.shape == vllm_output.shape
assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1))
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.exceptions import VLLMValidationError
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
max_model_len = 128
input_str = """Immerse yourself in the enchanting chronicle of calculus, a
mathematical domain that has radically transformed our comprehension of
change and motion. Despite its roots in ancient civilizations, the
formal birth of calculus predominantly occurred in the 17th century,
primarily under the influential guidance of Sir Isaac Newton and Gottfried
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
ancient Greek mathematics,most notably in the works of Eudoxus and
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
technique for computing areas and volumes through the use of finite sums.
This methodology laid crucial foundational work for integral calculus.
In the 17th century, both Newton and Leibniz independently pioneered
calculus, each contributing unique perspectives that would shape this new
field."""
@pytest.fixture(scope="module")
def vllm_model(vllm_runner):
with vllm_runner(
MODEL_NAME, runner="pooling", max_model_len=max_model_len
) as model:
yield model
def test_smaller_truncation_size(
vllm_model,
):
truncate_prompt_tokens = 10
vllm_output = vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)
prompt_tokens = vllm_output[0].prompt_token_ids
assert len(prompt_tokens) == truncate_prompt_tokens
def test_max_truncation_size(vllm_model):
truncate_prompt_tokens = -1
vllm_output = vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)
prompt_tokens = vllm_output[0].prompt_token_ids
# Pooling requests emit an embedding rather than generated tokens, so -1
# truncates to the full context length.
assert len(prompt_tokens) == max_model_len
def test_bigger_truncation_size(vllm_model):
truncate_prompt_tokens = max_model_len + 1
with pytest.raises(VLLMValidationError):
vllm_model.llm.embed(
input_str,
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
)
@@ -1,182 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Collective-safety tests for the Moondream3 MoE layer.
The MoE forward may fall back from the fused kernel to a Python expert loop.
That decision is made per rank from a rank-local exception, so every path has
to issue exactly one all-reduce. A path that issues two (or none) desyncs the
tensor-parallel group and wedges it for the rest of the run.
"""
import pytest
import torch
from tests.utils import ensure_current_vllm_config
from vllm.distributed.parallel_state import (
destroy_distributed_environment,
destroy_model_parallel,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.models import moondream3
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
pytestmark = pytest.mark.cpu_test
HIDDEN_SIZE = 8
EXPERT_INNER_DIM = 16
NUM_EXPERTS = 4
EXPERTS_PER_TOKEN = 2
NUM_TOKENS = 3
class _StubGate(torch.nn.Module):
"""Router stand-in, so the test does not depend on platform-specific
``ReplicatedLinear`` weight post-processing."""
def __init__(self, hidden_size: int, num_experts: int):
super().__init__()
self.linear = torch.nn.Linear(hidden_size, num_experts)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
return self.linear(x), None
class _CountingAllReduce:
"""Stand-in for tensor_model_parallel_all_reduce that counts calls."""
def __init__(self):
self.calls = 0
def __call__(self, x: torch.Tensor) -> torch.Tensor:
self.calls += 1
return x
@pytest.fixture
def tp1_cpu():
"""A single-rank CPU tensor-parallel group, enough to build the layer."""
update_environment_variables(
{
"RANK": "0",
"LOCAL_RANK": "0",
"WORLD_SIZE": "1",
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(get_open_port()),
}
)
init_distributed_environment(backend="gloo")
with ensure_current_vllm_config():
initialize_model_parallel(tensor_model_parallel_size=1)
yield
destroy_model_parallel()
destroy_distributed_environment()
@pytest.fixture
def moe(tp1_cpu) -> moondream3.Moondream3TextMoE:
"""A single-rank MoE layer with deterministic weights."""
layer = moondream3.Moondream3TextMoE(
hidden_size=HIDDEN_SIZE,
expert_inner_dim=EXPERT_INNER_DIM,
num_experts=NUM_EXPERTS,
experts_per_token=EXPERTS_PER_TOKEN,
)
layer.gate = _StubGate(HIDDEN_SIZE, NUM_EXPERTS)
torch.nn.init.normal_(layer.fc1_weight, std=0.02)
torch.nn.init.normal_(layer.fc2_weight, std=0.02)
return layer
@pytest.fixture
def all_reduce(monkeypatch: pytest.MonkeyPatch) -> _CountingAllReduce:
counter = _CountingAllReduce()
monkeypatch.setattr(moondream3, "tensor_model_parallel_all_reduce", counter)
return counter
def test_expert_loop_all_reduces_once(moe, all_reduce):
"""The Python expert loop issues exactly one all-reduce."""
moe._use_fused_moe = False
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
out = moe(x)
assert out.shape == x.shape
assert all_reduce.calls == 1
def test_fused_path_all_reduces_once(moe, all_reduce, monkeypatch):
"""The fused kernel path issues exactly one all-reduce."""
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
monkeypatch.setattr(
moondream3,
"fused_experts",
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
)
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
out = moe(x)
assert out.shape == x.shape
assert all_reduce.calls == 1
def test_all_reduce_failure_is_not_retried(moe, monkeypatch):
"""A failing all-reduce must propagate, not trigger a second collective.
Regression test. The all-reduce used to sit inside the ``try`` guarding
the fused kernel, so a ``RuntimeError`` raised by the collective itself
was caught, and this rank went on to run the fallback loop and all-reduce
a *second* time -- two collectives on one rank against one on each of its
peers, which desyncs the tensor-parallel group and wedges it for good.
"""
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
monkeypatch.setattr(
moondream3,
"fused_experts",
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
)
calls = []
def _failing_all_reduce(x: torch.Tensor) -> torch.Tensor:
calls.append(x)
raise RuntimeError("collective failed")
monkeypatch.setattr(
moondream3, "tensor_model_parallel_all_reduce", _failing_all_reduce
)
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
with pytest.raises(RuntimeError, match="collective failed"):
moe(x)
assert len(calls) == 1, (
"the failed all-reduce was retried on the fallback path; peers issued "
"one collective and this rank issued two"
)
@pytest.mark.parametrize("exc", [RuntimeError("boom"), NotImplementedError()])
def test_fallback_after_fused_failure_all_reduces_once(
moe, all_reduce, monkeypatch, exc
):
"""A rank whose fused kernel fails still all-reduces exactly once."""
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
def _raise(**kwargs):
raise exc
monkeypatch.setattr(moondream3, "fused_experts", _raise)
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
out = moe(x)
assert out.shape == x.shape
assert all_reduce.calls == 1
# The fused path is disabled for subsequent forwards...
assert moe._use_fused_moe is False
# ...and those still all-reduce exactly once each.
moe(x)
assert all_reduce.calls == 2
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.audio import AudioAsset
from vllm.model_executor.models.moss_audio import MOSS_AUDIO_PLACEHOLDER
from vllm.platforms import current_platform
from ...registry import HF_EXAMPLE_MODELS
from ...utils import check_logprobs_close
CORE_MODEL = pytest.param(
"OpenMOSS-Team/MOSS-Audio-4B-Instruct",
marks=pytest.mark.core_model,
id="4b-instruct",
)
EXTENDED_MODELS = [
"OpenMOSS-Team/MOSS-Audio-4B-Thinking",
"OpenMOSS-Team/MOSS-Audio-8B-Instruct",
"OpenMOSS-Team/MOSS-Audio-8B-Thinking",
]
ACCURACY_MODELS = [CORE_MODEL, *EXTENDED_MODELS]
PARALLEL_SMOKE_CASES = [
pytest.param({"tensor_parallel_size": 2}, id="tp2"),
pytest.param({"pipeline_parallel_size": 2}, id="pp2"),
pytest.param(
{"tensor_parallel_size": 2, "pipeline_parallel_size": 2},
id="tp2_pp2",
),
]
HF_ACCURACY_SKIP_REASON = (
"HF AutoModelForCausalLM cannot load remote MOSS-Audio configs; "
"vLLM generation coverage is provided by the smoke tests below."
)
@pytest.mark.core_model
def test_moss_audio_generation_smoke(vllm_runner) -> None:
model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
with vllm_runner(
model,
dtype="half",
enforce_eager=True,
max_model_len=1024,
limit_mm_per_prompt={"audio": 1},
trust_remote_code=True,
) as vllm_model:
outputs = vllm_model.generate_greedy(
prompts,
max_tokens=4,
audios=audios,
)
assert len(outputs) == 1
assert len(outputs[0][1]) > 0
@pytest.mark.skip(reason=HF_ACCURACY_SKIP_REASON)
@pytest.mark.parametrize("model", ACCURACY_MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [8])
@pytest.mark.parametrize("num_logprobs", [5])
def test_moss_audio_hf_vllm_accuracy(
hf_runner,
vllm_runner,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."]
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
with vllm_runner(
model,
dtype=dtype,
enforce_eager=True,
max_model_len=1024,
limit_mm_per_prompt={"audio": 1},
trust_remote_code=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=audios,
)
with hf_runner(model, dtype=dtype, trust_remote_code=True) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=audios,
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.core_model
@pytest.mark.parametrize("parallel_kwargs", PARALLEL_SMOKE_CASES)
def test_moss_audio_parallel_smoke(vllm_runner, parallel_kwargs) -> None:
model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
required_gpus = parallel_kwargs.get(
"tensor_parallel_size", 1
) * parallel_kwargs.get("pipeline_parallel_size", 1)
if current_platform.device_count() < required_gpus:
# TP/PP integration smoke runs on local or multi-GPU CI only.
pytest.skip(f"Requires at least {required_gpus} GPUs")
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
with vllm_runner(
model,
dtype="half",
enforce_eager=True,
max_model_len=1024,
limit_mm_per_prompt={"audio": 1},
trust_remote_code=True,
**parallel_kwargs,
) as vllm_model:
outputs = vllm_model.generate_greedy(
prompts,
max_tokens=4,
audios=audios,
)
assert len(outputs) == 1
assert len(outputs[0][1]) > 0
@@ -1,225 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import asdict
from typing import TYPE_CHECKING, Any
import pytest
import torch
from vllm import SamplingParams
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from ....utils import VLLM_PATH, large_gpu_test
from ...utils import check_logprobs_close
if TYPE_CHECKING:
from _typeshed import StrPath
PIXTRAL_ID = "mistralai/Pixtral-12B-2409"
MISTRAL_SMALL_3_1_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
MINISTRAL_3B_ID = "mistralai/Ministral-3-3B-Instruct-2512"
MODELS = [PIXTRAL_ID, MISTRAL_SMALL_3_1_ID]
IMG_URLS = [
"237-400x300.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
"231-200x300.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
"27-500x500.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
"17-150x600.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
]
PROMPT = "Describe each image in one short sentence."
def _create_msg_format(urls: list[str]) -> list[dict[str, Any]]:
return [
{
"role": "user",
"content": [
{
"type": "text",
"text": PROMPT,
}
]
+ [{"type": "image_url", "image_url": {"url": url}} for url in urls],
}
]
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
LIMIT_MM_PER_PROMPT = dict(image=4)
MAX_MODEL_LEN = [8192, 65536]
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
assert FIXTURES_PATH.exists()
FIXTURE_LOGPROBS_CHAT = {
PIXTRAL_ID: FIXTURES_PATH / "pixtral_chat.json",
MISTRAL_SMALL_3_1_ID: FIXTURES_PATH / "mistral_small_3_chat.json",
MINISTRAL_3B_ID: FIXTURES_PATH / "ministral_3b_chat.json",
}
OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]]
@pytest.mark.parametrize(
"backend",
[
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.TORCH_SDPA,
],
)
def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None:
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
[4, 6],
backend,
hidden_size=64,
tp_size=1,
device=torch.device("cpu"),
)
assert cu_seqlens.dtype == torch.int32
assert max_seqlen.dtype == torch.int32
if backend == AttentionBackendEnum.FLASHINFER:
assert max_seqlen.item() >= 6
assert sequence_lengths is not None
assert sequence_lengths.dtype == torch.int32
assert len(cu_seqlens) % 2 == 0
else:
expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0
assert max_seqlen.item() == expected_max_seqlen
assert cu_seqlens.tolist() == [0, 4, 10]
assert sequence_lengths is None
# For the test author to store golden output in JSON
def _dump_outputs_w_logprobs(
outputs: OutputsLogprobs,
filename: "StrPath",
) -> None:
json_data = [
(
tokens,
text,
[
{k: asdict(v) for k, v in token_logprobs.items()}
for token_logprobs in (logprobs or [])
],
)
for tokens, text, logprobs in outputs
]
with open(filename, "w") as f:
json.dump(json_data, f)
def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
with open(filename, "rb") as f:
json_data = json.load(f)
return [
(
tokens,
text,
[
{int(k): Logprob(**v) for k, v in token_logprobs.items()}
for token_logprobs in logprobs
],
)
for tokens, text, logprobs in json_data
]
@large_gpu_test(min_gb=80)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("max_model_len", MAX_MODEL_LEN)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_chat(
vllm_runner, max_model_len: int, model: str, dtype: str, local_asset_server
) -> None:
if (
model == MISTRAL_SMALL_3_1_ID
and max_model_len == 65536
and current_platform.is_rocm()
):
pytest.skip(
"OOM on ROCm: 24B model with 65536 context length exceeds GPU memory"
)
EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(FIXTURE_LOGPROBS_CHAT[model])
with vllm_runner(
model,
dtype=dtype,
tokenizer_mode="mistral",
load_format="mistral",
config_format="mistral",
max_model_len=max_model_len,
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
) as vllm_model:
outputs = []
urls_all = [local_asset_server.url_for(u) for u in IMG_URLS]
msgs = [
_create_msg_format(urls_all[:1]),
_create_msg_format(urls_all[:2]),
_create_msg_format(urls_all),
]
for msg in msgs:
output = vllm_model.llm.chat(msg, sampling_params=SAMPLING_PARAMS)
outputs.extend(output)
logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
# Remove last `None` prompt_logprobs to compare with fixture
for i in range(len(logprobs)):
assert logprobs[i][-1] is None
logprobs[i] = logprobs[i][:-1]
check_logprobs_close(
outputs_0_lst=EXPECTED_CHAT_LOGPROBS,
outputs_1_lst=logprobs,
name_0="h100_ref",
name_1="output",
)
@large_gpu_test(min_gb=16)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_chat_consolidated(vllm_runner, dtype: str, local_asset_server) -> None:
EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(
FIXTURE_LOGPROBS_CHAT[MINISTRAL_3B_ID]
)
with vllm_runner(
MINISTRAL_3B_ID,
dtype=dtype,
tokenizer_mode="mistral",
load_format="mistral",
config_format="mistral",
max_model_len=8192,
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
) as vllm_model:
outputs = []
urls_all = [local_asset_server.url_for(u) for u in IMG_URLS]
msgs = [
_create_msg_format(urls_all[:1]),
_create_msg_format(urls_all[:2]),
_create_msg_format(urls_all),
]
for msg in msgs:
output = vllm_model.llm.chat(msg, sampling_params=SAMPLING_PARAMS)
outputs.extend(output)
logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
for i in range(len(logprobs)):
assert logprobs[i][-1] is None
logprobs[i] = logprobs[i][:-1]
check_logprobs_close(
outputs_0_lst=EXPECTED_CHAT_LOGPROBS,
outputs_1_lst=logprobs,
name_0="h100_ref",
name_1="output",
)
@@ -1,357 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fractions import Fraction
import pytest
import pytest_asyncio
from mistral_common.protocol.transcription.request import (
StreamingMode,
TranscriptionRequest,
)
from mistral_common.tokens.tokenizers.audio import Audio
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
from vllm import LLM, SamplingParams
from vllm.assets.audio import AudioAsset
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.protocol import StreamingInput
from vllm.platforms import current_platform
from vllm.renderers.inputs.preprocess import parse_model_prompt
from vllm.utils.math_utils import cdiv
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.kv_cache_interface import SlidingWindowSpec
from ....utils import ROCM_ENGINE_KWARGS, create_new_process_for_each_test
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
ENGINE_CONFIG = {
"model": MODEL_NAME,
"max_model_len": 8192,
"max_num_seqs": 4,
"limit_mm_per_prompt": {"audio": 1},
"config_format": "mistral",
"load_format": "mistral",
"tokenizer_mode": "mistral",
"enforce_eager": True,
"gpu_memory_utilization": 0.9,
**ROCM_ENGINE_KWARGS,
}
EXPECTED_TEXT = [
(
" First words I spoke in the original phonograph. "
"A little piece of practical poetry. Mary had a little lamb,"
" its fleece was quite a slow, and everywhere that Mary went, "
"the lamb was sure to go."
),
(
" And the 0-1 pitch on the way to Edgar Martinez. Swung on"
" the line. Down the left field line for OBS. Here comes Joy. "
"Here is Junior to third base. They're going to wave him in. "
"The throw to the plate will be late. The Mariners are going"
" to play. For the American League Championship, "
"I don't believe it. It just continues. My, oh, my."
),
]
def _normalize(texts: list[str]) -> list[str]:
# The model occasionally transcribes "OBS" as "a base hit" and
# "oh, my" as "oh my", but both are acoustically valid. Normalise so
# the assertion is stable across runs and hardware.
texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
return texts
def assert_encoder_kv_cache_spec(engine: LLM) -> None:
vllm_config = engine.llm_engine.vllm_config
audio_config = vllm_config.model_config.hf_config.audio_config
kv_cache_specs_per_rank = engine.llm_engine.model_executor.get_kv_cache_specs()
assert len(kv_cache_specs_per_rank) == 1
kv_cache_specs = kv_cache_specs_per_rank[0]
assert AUDIO_LAYER_NAME in kv_cache_specs, kv_cache_specs.keys()
spec = kv_cache_specs[AUDIO_LAYER_NAME]
assert audio_config.sliding_window == 750
assert audio_config.block_pool_size == 4
assert isinstance(spec, SlidingWindowSpec)
assert spec.block_size == 16
assert spec.num_kv_heads == 32
assert spec.tokens_per_state == Fraction(1, 4)
# cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra
# +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec).
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
assert (
spec.max_admission_blocks_per_request(
max_in_flight_tokens=1,
max_model_len=vllm_config.model_config.max_model_len,
)
== 13
)
@pytest.fixture
def audio_assets() -> list[AudioAsset]:
return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
@pytest.fixture
def tokenizer() -> MistralTokenizer:
return MistralTokenizer.from_hf_hub(MODEL_NAME)
@pytest_asyncio.fixture
async def async_engine(request):
engine_config = {**ENGINE_CONFIG, **getattr(request, "param", {})}
gpu_memory_utilization = ENGINE_CONFIG.get("gpu_memory_utilization", 0.9)
from vllm.platforms import current_platform
if current_platform.is_rocm():
from tests.utils import wait_for_memory_to_settle
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
engine_args = AsyncEngineArgs(**engine_config)
llm = AsyncLLM.from_engine_args(engine_args)
try:
yield llm
finally:
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
llm.shutdown(timeout=shutdown_timeout)
del llm
import torch
torch._dynamo.reset()
from vllm.distributed import cleanup_dist_env_and_memory
cleanup_dist_env_and_memory()
from tests.utils import wait_for_memory_to_settle
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
def _build_inputs(
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
) -> tuple[list[dict], list[SamplingParams]]:
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
inputs = []
sampling_params = []
for audio_asset in audio_assets:
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
req = TranscriptionRequest(
audio=audio.to_base64(audio.format),
streaming=StreamingMode.OFFLINE,
language=None,
)
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
tokens = tokenized.tokens
audio_array = tokenized.audios[0].audio_array
num_samples = audio_array.shape[0]
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
inputs.append(
{
"multi_modal_data": {"audio": [(audio_array, None)]},
"prompt_token_ids": tokens,
}
)
return inputs, sampling_params
def _assert_expected_text(outputs) -> None:
texts = _normalize([out.outputs[0].text for out in outputs])
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
vllm_kwargs = {**ENGINE_CONFIG}
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
with vllm_runner(**vllm_kwargs) as vllm_model:
assert_encoder_kv_cache_spec(vllm_model.llm)
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
)
@pytest.mark.parametrize(
"cudagraph_mode",
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
)
@create_new_process_for_each_test(method="spawn")
def test_voxtral_realtime_cudagraph(
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
vllm_kwargs = {**ENGINE_CONFIG}
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
vllm_kwargs["enforce_eager"] = False
vllm_kwargs["compilation_config"] = {
"cudagraph_mode": cudagraph_mode,
# max_num_seqs is 4, so this is the only decode size worth capturing.
"cudagraph_capture_sizes": [4],
}
with vllm_runner(**vllm_kwargs) as vllm_model:
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
_assert_expected_text(outputs)
@pytest.mark.asyncio
async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine):
# Lazy import to avoid CUDA-reinitialization error
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
async def input_stream(audio_buffer):
async for prompt in audio_buffer.get_input_stream():
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
(engine_input,) = await async_engine.renderer.render_cmpl_async(
[parsed_prompt]
)
yield StreamingInput(prompt=engine_input)
output_tokens_list = []
for i, audio_asset in enumerate(audio_assets):
output_tokens = []
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
req = TranscriptionRequest(
streaming=StreamingMode.OFFLINE,
audio=audio.to_base64(audio.format),
language=None,
)
audio_enc = tokenizer.encode_transcription(req)
buffer = VoxtralRealtimeBuffer(audio_config, audio_enc.tokens)
await buffer.append_audio(audio_enc.audios[0].audio_array)
await buffer.append_audio(None)
request_id = f"session-{i}"
async for resp in async_engine.generate(
prompt=input_stream(buffer),
sampling_params=sampling_params,
request_id=request_id,
):
tokens = resp.outputs[0].token_ids[-1:]
output_tokens.extend(tokens)
await buffer.append_tokens(tokens)
output_tokens_list.append(output_tokens)
texts = _normalize(
[
tokenizer.decode(
output_tokens, special_token_policy=SpecialTokenPolicy.IGNORE
)
for output_tokens in output_tokens_list
]
)
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
assert got == expected, (
f"Output mismatch at index {i}:\n"
f" got: {got!r}\n"
f" expected: {expected!r}"
)
@pytest.mark.asyncio
async def test_voxtral_realtime_buffer_stops_at_max_model_len(tokenizer):
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
audio = Audio.from_file(AudioAsset("winning_call").get_local_path(), strict=False)
req = TranscriptionRequest(
streaming=StreamingMode.OFFLINE,
audio=audio.to_base64(audio.format),
language=None,
)
audio_enc = tokenizer.encode_transcription(req)
max_model_len = 64
buffer = VoxtralRealtimeBuffer(
audio_config, audio_enc.tokens, max_model_len=max_model_len
)
await buffer.append_audio(audio_enc.audios[0].audio_array)
await buffer.append_audio(None)
# Unlimited feedback tokens so only the length cap can stop the stream.
await buffer.append_tokens([0] * max_model_len)
n_tokens = 0
async for prompt in buffer.get_input_stream():
n_tokens += len(prompt["prompt_token_ids"])
assert len(audio_enc.tokens) < n_tokens < max_model_len
@pytest.mark.asyncio
@pytest.mark.parametrize("async_engine", [{"max_model_len": 128}], indirect=True)
async def test_voxtral_realtime_generator_max_model_len(
audio_assets, tokenizer, async_engine
):
# Lazy import to avoid CUDA-reinitialization error
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
max_model_len = async_engine.model_config.max_model_len
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
async def input_stream(audio_buffer):
async for prompt in audio_buffer.get_input_stream():
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
(engine_input,) = await async_engine.renderer.render_cmpl_async(
[parsed_prompt]
)
yield StreamingInput(prompt=engine_input)
# winning_call is ~300 tokens long, well past max_model_len.
audio = Audio.from_file(audio_assets[1].get_local_path(), strict=False)
req = TranscriptionRequest(
streaming=StreamingMode.OFFLINE,
audio=audio.to_base64(audio.format),
language=None,
)
audio_enc = tokenizer.encode_transcription(req)
buffer = VoxtralRealtimeBuffer(
audio_config, audio_enc.tokens, max_model_len=max_model_len
)
await buffer.append_audio(audio_enc.audios[0].audio_array)
await buffer.append_audio(None)
output_tokens = []
async for resp in async_engine.generate(
prompt=input_stream(buffer),
sampling_params=sampling_params,
request_id="session-max-len",
):
tokens = resp.outputs[0].token_ids[-1:]
output_tokens.extend(tokens)
await buffer.append_tokens(tokens)
assert resp.finished
assert 0 < len(output_tokens) < max_model_len
# The engine is still healthy after the session hit the cap.
assert not async_engine.errored
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any
import pytest
import torch
from transformers import AutoModelForSpeechSeq2Seq
from vllm.assets.audio import AudioAsset
from vllm.multimodal.audio import AudioResampler
from vllm.platforms import current_platform
from ....conftest import HfRunner, PromptAudioInput, VllmRunner
from ....utils import create_new_process_for_each_test, multi_gpu_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import check_logprobs_close
VLLM_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
HF_PROMPT = ""
# Whisper expects 16kHz audio
WHISPER_SAMPLE_RATE = 16000
BEAM_WIDTHS = (1, 2)
@pytest.fixture(autouse=True)
def use_spawn_for_whisper(monkeypatch):
"""Whisper has issues with forked workers, use spawn instead."""
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
def run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: Sequence[tuple[list[str], list[str], PromptAudioInput]],
model: str,
*,
max_model_len: int,
dtype: str,
max_tokens: int,
num_logprobs: int,
tensor_parallel_size: int,
distributed_executor_backend: str | None = None,
enforce_eager: bool = True,
gpu_memory_utilization: float = 0.9,
) -> None:
"""Inference result should be the same between hf and vllm.
All the audio fixtures for the test are from AudioAsset.
For huggingface runner, we provide the audio as input.
For vllm runner, we provide MultiModalDataDict objects
and corresponding MultiModalConfig as input.
"""
with vllm_runner(
model,
dtype=dtype,
max_model_len=max_model_len,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
limit_mm_per_prompt={"audio": 2},
enforce_eager=enforce_eager,
gpu_memory_utilization=gpu_memory_utilization,
disable_custom_all_reduce=True,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.generate_greedy_logprobs(
vllm_prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=audios,
)
for vllm_prompts, _, audios in inputs
]
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
hf_prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=audios,
)
for _, hf_prompts, audios in inputs
]
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.fixture
def resampled_assets() -> list[tuple[Any, int]]:
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
sampled_assets = []
resampler = AudioResampler(target_sr=WHISPER_SAMPLE_RATE)
for asset in audio_assets:
audio, orig_sr = asset.audio_and_sample_rate
# Resample to Whisper's expected sample rate (16kHz)
if orig_sr != WHISPER_SAMPLE_RATE:
audio = resampler.resample(audio, orig_sr=orig_sr)
sampled_assets.append(
(audio, WHISPER_SAMPLE_RATE),
)
return sampled_assets
@pytest.fixture
def input_audios(
resampled_assets,
) -> list[tuple[list[str], list[str], list[tuple[Any, int]]]]:
inputs = []
# audio assets are resampled to WHISPER_SAMPLE_RATE
for audio_info in resampled_assets:
# vLLM prompts, HF prompts, audio inputs
inputs.append(([VLLM_PROMPT], [HF_PROMPT], [audio_info]))
return inputs
def check_model_available(model: str) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [64])
def test_beam_search_encoder_decoder(
hf_runner,
vllm_runner,
dtype: str,
max_tokens: int,
resampled_assets,
) -> None:
"""Test beam search with encoder-decoder models (Whisper)."""
model = "openai/whisper-large-v3-turbo"
check_model_available(model)
hf_prompts = [
"<|startoftranscript|>",
"<|startoftranscript|>",
]
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
hf_outputs_by_beam_width = [
hf_model.generate_beam_search(
hf_prompts,
beam_width=beam_width,
max_tokens=max_tokens,
audios=resampled_assets,
)
for beam_width in BEAM_WIDTHS
]
# Test both explicit encoder/decoder prompts
vllm_prompts = [
# Implicit encoder/decoder prompt
{
"prompt": "<|startoftranscript|>",
"multi_modal_data": {"audio": resampled_assets[0]},
},
# Explicit encoder/decover prompt
{
"encoder_prompt": {
"prompt": "",
"multi_modal_data": {"audio": resampled_assets[1]},
},
"decoder_prompt": "<|startoftranscript|>",
},
]
with vllm_runner(
model,
dtype="half",
max_model_len=448,
tensor_parallel_size=1,
max_num_seqs=4,
limit_mm_per_prompt={"audio": 2},
enforce_eager=True,
) as vllm_model:
vllm_outputs_by_beam_width = [
vllm_model.generate_beam_search(
vllm_prompts,
beam_width=beam_width,
max_tokens=max_tokens,
)
for beam_width in BEAM_WIDTHS
]
for beam_width, hf_outputs, vllm_outputs in zip(
BEAM_WIDTHS, hf_outputs_by_beam_width, vllm_outputs_by_beam_width
):
for i in range(len(vllm_prompts)):
hf_output_ids, hf_output_texts = hf_outputs[i]
vllm_output_ids, vllm_output_texts = vllm_outputs[i]
for j, (hf_text, vllm_text) in enumerate(
zip(hf_output_texts, vllm_output_texts)
):
print(f">>>{j}-th hf output [NOTE: special tokens are filtered]:")
print(hf_text)
print(f">>>{j}-th vllm output:")
print(vllm_text)
# Check that we got the same number of beams
assert len(hf_output_ids) == len(vllm_output_ids) == beam_width
# For encoder-decoder models, we primarily want to verify that:
# 1. Beam search completes without errors
# 2. We get the expected number of beams
# 3. Outputs are reasonable (non-empty, diverse beams)
for j in range(len(vllm_output_ids)):
# Check that outputs are not empty
assert len(vllm_output_ids[j]) > 0, (
f"Prompt {i}, beam {j}: empty output"
)
# Check that decoded text is not empty
assert len(vllm_output_texts[j].strip()) > 0, (
f"Prompt {i}, beam {j}: empty text output"
)
def test_parse_language_detection_output():
"""Unit test for WhisperForConditionalGeneration.parse_language_detection_output.
No GPU or model loading required.
"""
from unittest.mock import MagicMock
from vllm.model_executor.models.whisper import (
WhisperForConditionalGeneration,
)
cls = WhisperForConditionalGeneration
def make_tokenizer(return_value: str) -> MagicMock:
tok = MagicMock()
tok.decode = MagicMock(return_value=return_value)
return tok
# English
assert (
cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en"
)
# German
assert (
cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de"
)
# Unsupported language code
with pytest.raises(AssertionError):
cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>"))
# No special token format
with pytest.raises(AssertionError):
cls.parse_language_detection_output([1], make_tokenizer("hello"))
# Empty token_ids
with pytest.raises((AssertionError, IndexError)):
cls.parse_language_detection_output([], make_tokenizer("anything"))
def test_create_fake_bias_for_k_proj_injects_out_features():
"""Packed k_proj weights get a zeros bias of length ``out_features``."""
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
packed = torch.ones(2, 16)
result = dict(
_create_fake_bias_for_k_proj(
[("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)],
".k_proj.weight_packed",
out_features=8,
)
)
bias_name = "model.encoder.layers.0.self_attn.k_proj.bias"
assert bias_name in result
assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result
assert tuple(result[bias_name].shape) == (8,)
assert torch.equal(result[bias_name], torch.zeros(8))
def test_create_fake_bias_for_k_proj_forwards_real_bias():
"""A checkpoint bias is forwarded and no extra fake bias is emitted."""
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
weight = torch.ones(8, 16)
real_bias = torch.ones(8)
items = list(
_create_fake_bias_for_k_proj(
[
("layers.0.self_attn.k_proj.weight", weight),
("layers.0.self_attn.k_proj.bias", real_bias),
],
".k_proj.weight",
out_features=8,
)
)
names = [n for n, _ in items]
assert names == [
"layers.0.self_attn.k_proj.weight",
"layers.0.self_attn.k_proj.bias",
]
assert items[1][1] is real_bias
@pytest.mark.core_model
@pytest.mark.cpu_model
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
@pytest.mark.parametrize("dtype", ["half", "float"])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("enforce_eager", [True, False])
def test_models(
hf_runner,
vllm_runner,
model: str,
dtype: str,
num_logprobs: int,
input_audios,
enforce_eager: bool,
) -> None:
check_model_available(model)
if current_platform.is_cpu() and not enforce_eager:
pytest.skip("Skipping test for CPU with non-eager mode")
run_test(
hf_runner,
vllm_runner,
input_audios,
model,
dtype=dtype,
max_model_len=448,
max_tokens=200,
num_logprobs=num_logprobs,
tensor_parallel_size=1,
enforce_eager=enforce_eager,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.core_model
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [200])
@pytest.mark.parametrize("num_logprobs", [5])
@create_new_process_for_each_test("spawn")
def test_models_distributed(
hf_runner,
vllm_runner,
model: str,
distributed_executor_backend: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
input_audios,
) -> None:
check_model_available(model)
run_test(
hf_runner,
vllm_runner,
input_audios,
model,
dtype=dtype,
max_model_len=448,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
tensor_parallel_size=2,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=False,
gpu_memory_utilization=0.65,
)
@pytest.mark.core_model
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
def test_encoder_cache_cleanup(
vllm_runner,
model: str,
input_audios,
monkeypatch,
) -> None:
"""Test that encoder cache is properly cleaned up after requests complete.
This is a regression test for a bug where encoder cache entries were freed
in the same scheduling step they were allocated, before the model could use
them.
"""
# Set single-process mode to access the model runner's encoder cache directly
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
check_model_available(model)
with vllm_runner(
model,
dtype="half",
max_model_len=448,
tensor_parallel_size=1,
limit_mm_per_prompt={"audio": 2},
enforce_eager=True,
) as vllm_model:
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
encoder_cache = model_runner.encoder_cache
# Run multiple sequential requests to ensure cache is properly managed
for vllm_prompts, _, audios in input_audios:
vllm_model.generate_greedy(vllm_prompts, max_tokens=50, audios=audios)
# After all requests complete, encoder cache should be empty
cache_size = len(encoder_cache)
assert cache_size == 0, (
f"Encoder cache should be empty after all requests complete, "
f"but has {cache_size} entries. This indicates encoder cache "
f"entries are not being properly freed."
)
File diff suppressed because it is too large Load Diff
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Pytest configuration for vLLM multimodal pooling tests."""
import gc
import pytest
import torch
from tests.utils import wait_for_memory_to_settle
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
@pytest.fixture(autouse=True)
def release_gpu_memory_between_tests():
"""Reclaim cached GPU memory between tests in this shard.
These tests start many vLLM/HF engines sequentially in one pytest
process; without explicit reclamation between tests, VRAM fragmentation
accumulates and later tests fail at engine startup ("Engine core
initialization failed").
"""
yield
gc.collect()
if torch.accelerator.is_available():
torch.accelerator.empty_cache()
if current_platform.is_rocm():
return
try:
wait_for_memory_to_settle()
except ValueError as e:
# Longer-lived (class/module-scoped) engine fixtures may legitimately
# still hold VRAM; nothing further to reclaim in that case.
logger.info("Failed to clean GPU memory: %s", e)
@@ -1,174 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
which loops over questions where the pooler batches them."""
import math
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.models.clef_joint_schema import (
JointSchemaHead,
JointSchemaPooler,
)
from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
DEVICE_TYPE = current_platform.device_type
HIDDEN, VOCAB = 64, 500
TYPES = [0, 1, 1, 2]
COUNTS = [2, 3, 1, 5]
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
for one record, with ``spans`` the option spans of each question."""
hidden = head.hidden_norm(hidden)
memory = head.memory_projection(hidden).unsqueeze(0)
global_vector = hidden[-1]
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
lexicals = [
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
]
options = torch.cat(
[
head.option_context_projection(c)
+ head.option_lexical_projection(lex)
+ head.option_question_projection(questions[i]).unsqueeze(0)
for i, (c, lex) in enumerate(zip(contexts, lexicals))
]
).unsqueeze(0)
for layer in head.evidence_layers:
options = layer(options, memory)
routed = torch.split(options[0], [len(q) for q in spans])
base = head.question_projection(questions)
summaries = []
for field, opts in zip(base, routed):
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
summaries.append((weights[:, None] * opts).sum(0))
fields = (
base
+ head.option_summary_norm(torch.stack(summaries))
+ head.global_projection(global_vector)
+ head.type_embedding(torch.tensor(TYPES, device=hidden.device))
).unsqueeze(0)
for layer in head.layers:
fields = layer(fields, memory)
fields = head.field_norm(fields[0])
logits = []
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
anchor = F.normalize(questions[i] + global_vector, dim=-1)
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
opts = head.option_norm(opts)
field = field.expand_as(opts)
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
joint = joint + head.residual_scorer(features).squeeze(-1)
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
return logits
@pytest.fixture
def pooler():
torch.manual_seed(0)
head = JointSchemaHead(
hidden_size=HIDDEN,
width=32,
routing_layers=2,
layers=2,
heads=4,
feedforward=48,
).eval()
with torch.no_grad():
for p in head.parameters():
p.add_(torch.randn_like(p) * 0.2)
head.to(DEVICE_TYPE)
embedding = torch.randn(VOCAB, HIDDEN, device=DEVICE_TYPE)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
enable_chunked_prefill=False, async_scheduling=False
)
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=config,
):
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
return pooler, embedding
def schema(length: int):
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
pos = 20
question_spans: list[tuple[int, int]] = []
spans: list[list[tuple[int, int]]] = []
for count in COUNTS:
question_spans.append((pos, pos + 4))
pos += 6
spans.append([])
for k in range(count):
n = 1 + (k * 3) % 4
spans[-1].append((pos, pos + n))
pos += n + 1
assert pos < length
extra = {
"question_types": TYPES,
"question_spans": [[s - length, e - length] for s, e in question_spans],
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
"option_counts": COUNTS,
}
return {"joint_schema": extra}, question_spans, spans
@torch.inference_mode()
def test_pooler_matches_reference(pooler):
pooler, embedding = pooler
lengths = [120, 90, 100]
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
hidden = torch.randn(sum(lengths), HIDDEN, device=DEVICE_TYPE)
first, first_questions, first_spans = schema(lengths[0])
last, last_questions, last_spans = schema(lengths[2])
params = [
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
# A prompt without a schema, like the profiling run's.
PoolingParams(task="token_classify", use_activation=True),
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
]
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
for i, ids in enumerate(token_ids):
padded[i, : len(ids)] = ids
metadata = PoolingMetadata(
prompt_lens=torch.tensor(lengths),
prompt_token_ids=None,
prompt_token_ids_cpu=padded,
pooling_params=params,
pooling_states=[PoolingStates() for _ in lengths],
)
metadata.build_pooling_cursor(
torch.tensor(lengths).numpy(),
seq_lens_cpu=torch.tensor(lengths),
device=hidden.device,
)
probs, empty, logits = pooler(hidden, metadata)
first_hidden, _, last_hidden = torch.split(hidden, lengths)
expected = reference_logits(
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
)
expected_probs = torch.cat([q.softmax(-1) for q in expected])
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
assert empty.numel() == 0
expected = reference_logits(
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
)
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
@@ -1,138 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import CLIPModel
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ...utils import check_embeddings_close
HF_TEXT_PROMPTS = [
"a photo of a stop sign",
"a photo of a cherry blossom",
]
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "",
"cherry_blossom": "",
}
)
MODELS = ["openai/clip-vit-base-patch32"]
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_cases: list[tuple[list[str], PromptImageInput]],
model: str,
*,
dtype: str,
) -> None:
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
with vllm_runner(
model, runner="pooling", dtype=dtype, enforce_eager=True, max_model_len=77
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.embed(input_texts, images=input_images)
for input_texts, input_images in input_cases
]
texts = [HF_TEXT_PROMPTS[0]]
images = [input_cases[1][1][0]]
with pytest.raises(ValueError, match="not both"):
vllm_model.embed(texts, images=images)
# Should still be able to run subsequent requests
vllm_model.embed(texts)
vllm_model.embed([""], images=images)
# Mixed image+text batch must not skip the text encoder (#53091).
mixed_outputs = vllm_outputs_per_case[2]
check_embeddings_close(
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
embeddings_1_lst=[mixed_outputs[0]],
name_0="text_only",
name_1="mixed_text",
)
check_embeddings_close(
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
embeddings_1_lst=[mixed_outputs[1]],
name_0="image_only",
name_1="mixed_image",
)
empty_text = vllm_model.embed([""])
empty_sim = torch.nn.functional.cosine_similarity(
torch.tensor(mixed_outputs[0]),
torch.tensor(empty_text[0]),
dim=0,
)
assert empty_sim < 0.99, (
"Mixed-batch text embedding collapsed to the empty-string vector "
f"(cosine={empty_sim:.4f})"
)
with hf_runner(model, dtype=dtype, auto_cls=CLIPModel) as hf_model:
hf_outputs_per_case = []
for input_texts, input_images in input_cases:
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
hf_outputs = []
for inputs in all_inputs:
inputs = hf_model.wrap_device(inputs)
if "pixel_values" in inputs:
pooled_output = hf_model.model.get_image_features(
pixel_values=inputs.pixel_values,
)
else:
pooled_output = hf_model.model.get_text_features(
input_ids=inputs.input_ids,
attention_mask=inputs.attention_mask,
)
if not isinstance(pooled_output, torch.Tensor):
pooled_output = pooled_output.pooler_output
pooled_output = pooled_output.squeeze(0)
hf_outputs.append(pooled_output.tolist())
hf_outputs_per_case.append(hf_outputs)
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["float"])
def test_models(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
text_images = [None] * len(HF_TEXT_PROMPTS)
images = [asset.pil_image for asset in image_assets]
input_cases = [
(HF_TEXT_PROMPTS, text_images),
(HF_IMAGE_PROMPTS, images),
([HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]], [None, images[0]]),
]
_run_test(
hf_runner,
vllm_runner,
input_cases,
model,
dtype=dtype,
)
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColModernVBERT multimodal late-interaction model.
ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder
with a pixel shuffle connector and ColBERT-style 128-dim per-token
embeddings for visual document retrieval.
"""
import pytest
import torch
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
MODEL_NAME = "ModernVBERT/colmodernvbert-merged"
COLBERT_DIM = 128
DTYPE = "half"
@pytest.fixture(scope="module")
def colmodernvbert_model(vllm_runner):
with vllm_runner(
MODEL_NAME,
runner="pooling",
dtype=DTYPE,
enforce_eager=True,
) as vllm_model:
yield vllm_model
# -----------------------------------------------------------------------
# Text-only tests
# -----------------------------------------------------------------------
def test_colmodernvbert_text_token_embed(colmodernvbert_model):
"""Text query produces per-token embeddings with shape (seq_len, 128)."""
outputs = colmodernvbert_model.token_embed(["What is machine learning?"])
assert len(outputs) == 1
emb = torch.tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == COLBERT_DIM
assert emb.shape[0] > 1
def test_colmodernvbert_text_relevance_ordering(colmodernvbert_model):
"""Relevant documents score higher than irrelevant ones."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"The weather in Paris is mild in spring.",
]
scores = colmodernvbert_model.score(query, documents)
assert len(scores) == 2
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
def test_colmodernvbert_text_late_interaction(colmodernvbert_model):
"""MaxSim scoring via vLLM matches manual computation."""
query = "What is the capital of France?"
doc = "The capital of France is Paris."
q_out = colmodernvbert_model.token_embed([query])
d_out = colmodernvbert_model.token_embed([doc])
q_emb = torch.tensor(q_out[0])
d_emb = torch.tensor(d_out[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = colmodernvbert_model.score(query, doc)
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
# -----------------------------------------------------------------------
# Image tests
# -----------------------------------------------------------------------
def test_colmodernvbert_image_token_embed(colmodernvbert_model, image_assets):
"""Image input produces per-token embeddings including vision tokens."""
image = image_assets[0].pil_image
inputs = colmodernvbert_model.get_inputs(
[""],
images=[image],
)
req_outputs = colmodernvbert_model.llm.encode(
inputs,
pooling_task="token_embed",
)
outputs = [req_output.outputs.data for req_output in req_outputs]
assert len(outputs) == 1
emb = torch.tensor(outputs[0])
assert emb.dim() == 2
assert emb.shape[1] == COLBERT_DIM
# Should have at least the image tokens (64 after pixel shuffle)
assert emb.shape[0] >= 64
@@ -1,244 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColPali late interaction model for multi-modal retrieval.
ColPali is a multi-vector retrieval model based on PaliGemma backbone
(SigLIP + Gemma) with ColBERT-style late interaction scoring (MaxSim).
It produces per-token embeddings for both text and image inputs.
"""
from io import BytesIO
import pybase64 as base64
import pytest
import torch
from PIL import Image
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from ....conftest import VllmRunner
MODELS = [
"vidore/colpali-v1.3-hf",
]
EMBED_DIMS = {
"vidore/colpali-v1.3-hf": 128,
}
TEXT_QUERIES = [
"What is the capital of France?",
"Describe the contents of the document.",
]
TEXT_DOCUMENTS = [
"The capital of France is Paris.",
"This document contains important financial data.",
]
DTYPE = "half"
GPU_MEMORY_UTILIZATION = 0.7
def _make_base64_image(
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
) -> str:
"""Create a small solid-color PNG image and return its base64 data URI."""
img = Image.new("RGB", (width, height), color)
buf = BytesIO()
img.save(buf, format="PNG")
b64 = base64.b64encode(buf.getvalue()).decode()
return f"data:image/png;base64,{b64}"
def _make_image_mm_param(
image_uri: str,
text: str | None = None,
) -> ScoreMultiModalParam:
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
content: list = [
ChatCompletionContentPartImageParam(
type="image_url",
image_url={"url": image_uri},
),
]
if text is not None:
content.append(
ChatCompletionContentPartTextParam(type="text", text=text),
)
return ScoreMultiModalParam(content=content)
def _run_token_embed_test(
vllm_model: VllmRunner,
model: str,
) -> None:
"""Verify per-token embedding shape and L2 normalization."""
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
assert len(outputs) == 1
emb = torch.tensor(outputs[0])
# Token embeddings should be 2D: [num_tokens, embed_dim]
assert emb.dim() == 2
assert emb.shape[1] == EMBED_DIMS[model]
assert emb.shape[0] > 1
# Verify L2 normalization
norms = torch.norm(emb, p=2, dim=-1)
torch.testing.assert_close(
norms,
torch.ones_like(norms),
rtol=1e-2,
atol=1e-2,
)
def _run_late_interaction_test(
vllm_model: VllmRunner,
) -> None:
"""Verify MaxSim scoring matches manual computation."""
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
q_emb = torch.tensor(q_outputs[0])
d_emb = torch.tensor(d_outputs[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def _run_relevance_test(
vllm_model: VllmRunner,
) -> None:
"""Verify that relevant documents score higher than irrelevant ones."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"The weather forecast shows rain tomorrow.",
"Deep learning uses neural networks for complex tasks.",
]
scores = vllm_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
# ── Multimodal scoring tests ────────────────────────────────
def _run_multimodal_text_query_image_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score a text query against image documents via the multimodal path."""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
blue_image = _make_base64_image(64, 64, color=(0, 0, 255))
query = "Describe the red object"
image_docs = [
_make_image_mm_param(red_image),
_make_image_mm_param(blue_image),
]
scores = vllm_model.llm.score(query, image_docs)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
def _run_multimodal_mixed_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score a text query against a mix of text and image documents."""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
query = "What is the capital of France?"
documents: list = [
"The capital of France is Paris.",
_make_image_mm_param(red_image),
]
scores = vllm_model.llm.score(query, documents)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
# Text document about France should score higher than a random image
assert scores[0].outputs.score > scores[1].outputs.score
def _run_multimodal_image_query_text_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score an image query against text documents."""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
image_query = _make_image_mm_param(red_image, text="red color")
documents = [
"A bright red sports car.",
"The weather forecast shows rain tomorrow.",
]
scores = vllm_model.llm.score(image_query, documents)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", [DTYPE])
def test_colpali_default_runner(
vllm_runner,
model: str,
dtype: str,
) -> None:
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
max_model_len=4096,
enforce_eager=True,
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
) as vllm_model:
_run_token_embed_test(vllm_model, model)
_run_late_interaction_test(vllm_model)
_run_relevance_test(vllm_model)
_run_multimodal_mixed_docs_test(vllm_model)
_run_multimodal_image_query_text_docs_test(vllm_model)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", [DTYPE])
def test_colpali_v2_multimodal_text_query_image_docs(
vllm_runner,
monkeypatch: pytest.MonkeyPatch,
model: str,
dtype: str,
) -> None:
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
attention_backend = "FLASH_ATTN" if current_platform.is_cuda() else None
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
max_model_len=4096,
enforce_eager=True,
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
attention_backend=attention_backend,
kernel_config={"enable_flashinfer_autotune": False},
) as vllm_model:
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
_run_multimodal_text_query_image_docs_test(vllm_model)
@@ -1,268 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColQwen3 late interaction model for multi-modal retrieval.
ColQwen3 is a multi-vector retrieval model based on Qwen3-VL backbone with
ColBERT-style late interaction scoring (MaxSim). It produces per-token
embeddings for both text and image inputs.
"""
from io import BytesIO
import pybase64 as base64
import pytest
import torch
from PIL import Image
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from ....conftest import VllmRunner
MODELS = [
"TomoroAI/tomoro-colqwen3-embed-4b",
"OpenSearch-AI/Ops-Colqwen3-4B",
"nvidia/nemotron-colembed-vl-4b-v2",
]
EMBED_DIMS = {
"TomoroAI/tomoro-colqwen3-embed-4b": 320,
"OpenSearch-AI/Ops-Colqwen3-4B": 2560,
"nvidia/nemotron-colembed-vl-4b-v2": 2560,
}
TEXT_QUERIES = [
"What is the capital of France?",
"Describe the contents of the document.",
]
TEXT_DOCUMENTS = [
"The capital of France is Paris.",
"This document contains important financial data.",
]
DTYPE = "half"
# The Tomoro model needs room for its vision encoder and a 4096-token KV cache
# on the 16 GiB devices used by the H200 MIG test lane.
GPU_MEMORY_UTILIZATION = 0.8
@pytest.fixture(scope="module", params=MODELS)
def colqwen3_model(request, vllm_runner):
model = request.param
with vllm_runner(
model,
runner="pooling",
dtype=DTYPE,
max_model_len=4096,
enforce_eager=True,
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
) as vllm_model:
yield model, vllm_model
def _make_base64_image(
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
) -> str:
"""Create a small solid-color PNG image and return its base64 data URI."""
img = Image.new("RGB", (width, height), color)
buf = BytesIO()
img.save(buf, format="PNG")
b64 = base64.b64encode(buf.getvalue()).decode()
return f"data:image/png;base64,{b64}"
def _make_image_mm_param(
image_uri: str,
text: str | None = None,
) -> ScoreMultiModalParam:
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
content: list = [
ChatCompletionContentPartImageParam(
type="image_url",
image_url={"url": image_uri},
),
]
if text is not None:
content.append(
ChatCompletionContentPartTextParam(type="text", text=text),
)
return ScoreMultiModalParam(content=content)
def _make_text_mm_param(text: str) -> ScoreMultiModalParam:
"""Build a ScoreMultiModalParam containing only text."""
return ScoreMultiModalParam(
content=[ChatCompletionContentPartTextParam(type="text", text=text)],
)
def _run_token_embed_test(
vllm_model: VllmRunner,
model: str,
) -> None:
"""Verify per-token embedding shape and L2 normalization."""
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
assert len(outputs) == 1
emb = torch.tensor(outputs[0])
# Token embeddings should be 2D: [num_tokens, embed_dim]
assert emb.dim() == 2
assert emb.shape[1] == EMBED_DIMS[model]
assert emb.shape[0] > 1
# Verify L2 normalization
norms = torch.norm(emb, p=2, dim=-1)
torch.testing.assert_close(
norms,
torch.ones_like(norms),
rtol=1e-2,
atol=1e-2,
)
def _run_late_interaction_test(
vllm_model: VllmRunner,
) -> None:
"""Verify MaxSim scoring matches manual computation."""
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
q_emb = torch.tensor(q_outputs[0])
d_emb = torch.tensor(d_outputs[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def _run_relevance_test(
vllm_model: VllmRunner,
) -> None:
"""Verify that relevant documents score higher than irrelevant ones."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"The weather forecast shows rain tomorrow.",
"Deep learning uses neural networks for complex tasks.",
]
scores = vllm_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
def test_colqwen3_token_embed(colqwen3_model) -> None:
model, vllm_model = colqwen3_model
_run_token_embed_test(vllm_model, model)
def test_colqwen3_late_interaction_scoring(colqwen3_model) -> None:
_, vllm_model = colqwen3_model
_run_late_interaction_test(vllm_model)
def test_colqwen3_relevance_ordering(colqwen3_model) -> None:
_, vllm_model = colqwen3_model
_run_relevance_test(vllm_model)
# ── Multimodal scoring tests ────────────────────────────────
def _run_multimodal_text_query_image_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score a text query against image documents via the multimodal path.
Verifies that score_data_to_prompts correctly handles image content
and produces valid MaxSim scores.
"""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
blue_image = _make_base64_image(64, 64, color=(0, 0, 255))
query = "Describe the red object"
image_docs = [
_make_image_mm_param(red_image),
_make_image_mm_param(blue_image),
]
scores = vllm_model.llm.score(query, image_docs)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
def _run_multimodal_mixed_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score a text query against a mix of text and image documents.
Ensures the late-interaction path handles heterogeneous document
types (plain strings alongside ScoreMultiModalParam images) in
a single call.
"""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
query = "What is the capital of France?"
documents: list = [
"The capital of France is Paris.",
_make_image_mm_param(red_image),
]
scores = vllm_model.llm.score(query, documents)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
# Text document about France should score higher than a random image
assert scores[0].outputs.score > scores[1].outputs.score
def _run_multimodal_image_query_text_docs_test(
vllm_model: VllmRunner,
) -> None:
"""Score an image query against text documents.
Verifies the reverse direction: multimodal query with text-only
documents through the late-interaction scoring path.
"""
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
image_query = _make_image_mm_param(red_image, text="red color")
documents = [
"A bright red sports car.",
"The weather forecast shows rain tomorrow.",
]
scores = vllm_model.llm.score(image_query, documents)
assert len(scores) == 2
for s in scores:
assert isinstance(s.outputs.score, float)
def test_colqwen3_multimodal_text_query_image_docs(colqwen3_model) -> None:
_, vllm_model = colqwen3_model
_run_multimodal_text_query_image_docs_test(vllm_model)
def test_colqwen3_multimodal_mixed_docs(colqwen3_model) -> None:
_, vllm_model = colqwen3_model
_run_multimodal_mixed_docs_test(vllm_model)
def test_colqwen3_multimodal_image_query_text_docs(colqwen3_model) -> None:
_, vllm_model = colqwen3_model
_run_multimodal_image_query_text_docs_test(vllm_model)
@@ -1,236 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ColQwen3.5 late interaction model for multi-modal retrieval.
ColQwen3.5 is a multi-vector retrieval model based on Qwen3.5 backbone with
ColBERT-style late interaction scoring (MaxSim). It produces per-token
embeddings for both text and image inputs.
"""
from types import SimpleNamespace
import pytest
import torch
from ....conftest import VllmRunner
MODELS = [
"athrael-soju/colqwen3.5-4.5B-v3",
]
EMBED_DIMS = {
"athrael-soju/colqwen3.5-4.5B-v3": 320,
}
TEXT_QUERIES = [
"What is the capital of France?",
"Describe the contents of the document.",
]
TEXT_DOCUMENTS = [
"The capital of France is Paris.",
"This document contains important financial data.",
]
DTYPE = "half"
@pytest.fixture(scope="module", params=MODELS)
def colqwen3_5_model(request, vllm_runner):
model = request.param
with vllm_runner(
model,
runner="pooling",
dtype=DTYPE,
max_model_len=4096,
enforce_eager=True,
) as vllm_model:
yield model, vllm_model
def _run_token_embed_test(
vllm_model: VllmRunner,
model: str,
) -> None:
"""Verify per-token embedding shape and L2 normalization."""
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
assert len(outputs) == 1
emb = torch.tensor(outputs[0])
# Token embeddings should be 2D: [num_tokens, embed_dim]
assert emb.dim() == 2
assert emb.shape[1] == EMBED_DIMS[model]
assert emb.shape[0] > 1
# Verify L2 normalization
norms = torch.norm(emb, p=2, dim=-1)
torch.testing.assert_close(
norms,
torch.ones_like(norms),
rtol=1e-2,
atol=1e-2,
)
def _run_late_interaction_test(
vllm_model: VllmRunner,
) -> None:
"""Verify MaxSim scoring matches manual computation."""
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
q_emb = torch.tensor(q_outputs[0])
d_emb = torch.tensor(d_outputs[0])
manual_score = compute_maxsim_score(q_emb, d_emb).item()
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
assert len(vllm_scores) == 1
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
def _run_relevance_test(
vllm_model: VllmRunner,
) -> None:
"""Verify that relevant documents score higher than irrelevant ones."""
query = "What is machine learning?"
documents = [
"Machine learning is a subset of artificial intelligence.",
"The weather forecast shows rain tomorrow.",
"Deep learning uses neural networks for complex tasks.",
]
scores = vllm_model.score(query, documents)
assert len(scores) == 3
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
def test_colqwen3_5_token_embed(colqwen3_5_model) -> None:
model, vllm_model = colqwen3_5_model
_run_token_embed_test(vllm_model, model)
def test_colqwen3_5_late_interaction_scoring(colqwen3_5_model) -> None:
_, vllm_model = colqwen3_5_model
_run_late_interaction_test(vllm_model)
def test_colqwen3_5_relevance_ordering(colqwen3_5_model) -> None:
_, vllm_model = colqwen3_5_model
_run_relevance_test(vllm_model)
@pytest.mark.parametrize(
("contract", "expected_is_causal"),
[("causal", True), ("bidirectional", False)],
)
def test_colqwen3_5_config_applies_declared_attention_contract(
contract: str,
expected_is_causal: bool,
) -> None:
from vllm.model_executor.models.config import (
MODELS_CONFIG_MAP,
ColQwen3_5Config,
)
assert MODELS_CONFIG_MAP["ColQwen3_5"] is ColQwen3_5Config
hf_config = SimpleNamespace(retrieval_attention_contract=contract)
text_config = SimpleNamespace()
model_config = SimpleNamespace(
hf_config=hf_config,
hf_text_config=text_config,
)
ColQwen3_5Config.verify_and_update_model_config(model_config)
assert hf_config.is_causal is expected_is_causal
assert text_config.is_causal is expected_is_causal
@pytest.mark.parametrize(
"hf_config",
[
SimpleNamespace(),
SimpleNamespace(retrieval_attention_contract="unsupported"),
SimpleNamespace(
retrieval_attention_contract="causal",
text_config=SimpleNamespace(retrieval_attention_contract="bidirectional"),
),
],
)
def test_colqwen3_5_config_rejects_invalid_attention_contract(hf_config) -> None:
from vllm.model_executor.models.config import ColQwen3_5Config
text_config = getattr(hf_config, "text_config", SimpleNamespace())
model_config = SimpleNamespace(
hf_config=hf_config,
hf_text_config=text_config,
)
with pytest.raises(ValueError, match="retrieval_attention_contract"):
ColQwen3_5Config.verify_and_update_model_config(model_config)
def test_colqwen3_5_bidirectional_contract_builds_encoder_only_attention(
monkeypatch,
) -> None:
from vllm.model_executor.models import qwen3_next
from vllm.model_executor.models.config import ColQwen3_5Config
from vllm.v1.attention.backend import AttentionType
hf_config = SimpleNamespace(retrieval_attention_contract="bidirectional")
text_config = SimpleNamespace(
hidden_size=256,
num_attention_heads=2,
num_key_value_heads=1,
head_dim=128,
max_position_embeddings=4096,
rope_parameters={},
rms_norm_eps=1e-6,
)
model_config = SimpleNamespace(
hf_config=hf_config,
hf_text_config=text_config,
)
ColQwen3_5Config.verify_and_update_model_config(model_config)
captured = {}
class FakeAttention(torch.nn.Module):
def __init__(self, *args, **kwargs) -> None:
super().__init__()
captured["attn_type"] = kwargs["attn_type"]
monkeypatch.setattr(qwen3_next, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(
qwen3_next, "QKVParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
)
monkeypatch.setattr(
qwen3_next, "RowParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
)
monkeypatch.setattr(
qwen3_next,
"get_rope",
lambda *args, **kwargs: SimpleNamespace(is_neox_style=False),
)
monkeypatch.setattr(
qwen3_next, "Qwen3NextRMSNorm", lambda *args, **kwargs: torch.nn.Identity()
)
monkeypatch.setattr(qwen3_next, "Attention", FakeAttention)
qwen3_next.Qwen3NextAttention(text_config)
assert captured["attn_type"] is AttentionType.ENCODER_ONLY
def test_colqwen3_5_encoder_only_attention_has_no_kv_cache_spec() -> None:
from vllm.model_executor.layers.attention import Attention
from vllm.v1.attention.backend import AttentionType
attention = SimpleNamespace(attn_type=AttentionType.ENCODER_ONLY)
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=16))
assert Attention.get_kv_cache_spec(attention, vllm_config) is None
@@ -1,87 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
The runtime reads a question's options from the 255 code logits of its readout
at the last prompt token. vLLM's classify head must give the same code logits
for the same prompt tokens, and for an image prompt that vLLM processes itself.
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
weight file without an index, next to the readout file.
"""
import pytest
import torch
from transformers import AutoModel
from vllm.assets.image import ImageAsset
from vllm.distributed import cleanup_dist_env_and_memory
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
STATE = (
"The order arrived damaged yesterday. The customer has a receipt and asks "
"for a replacement today."
)
QUESTIONS = {
"route": {
"type": "choice",
"instructions": "Which team should handle this request?",
"criteria": {
"returns": "Refunds, replacements and damaged deliveries",
"billing": "Payments, invoices and charges",
"technical": "Product setup and faults",
},
},
"receipt": {"type": "noul", "instructions": "Does the customer have a receipt?"},
"urgency": {
"type": "score",
"instructions": "How urgent is this request?",
"criteria": ["Routine", "Soon", "Today"],
},
"department": {
"type": "choice",
"criteria": {f"department {i}": None for i in range(40)},
},
}
IMAGE_QUESTIONS = {"photo": {"type": "noul", "instructions": "Is this a photograph?"}}
def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
return torch.softmax(logits[:count].float(), dim=-1)
@pytest.mark.parametrize("model", MODELS)
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
runtime = reference.runtime
text = runtime.prepare(STATE, QUESTIONS)
keys = text.runnable
counts = [len(text.questions[key].keys) for key in keys]
sequences = [text.sequences[key] for key in keys]
ref_logits = list(runtime.logits(sequences, counts).cpu())
image = ImageAsset("stop_sign").pil_image
visual = runtime.prepare("Describe the attached image.", IMAGE_QUESTIONS, [image])
(image_key,) = visual.runnable
image_prompt = visual.texts[image_key]
ref_logits += list(
runtime.image_logits(
[image_prompt], visual.images, [2], visual.lengths[image_key]
).cpu()
)
counts.append(2)
del reference, runtime
cleanup_dist_env_and_memory()
with vllm_runner(model, runner="pooling", max_model_len=4096) as vllm_model:
outputs = vllm_model.llm.classify(
[{"prompt_token_ids": ids} for ids in sequences]
+ [{"prompt": image_prompt, "multi_modal_data": {"image": image}}],
use_tqdm=False,
)
# The head returns raw code logits by default.
vllm_logits = [torch.tensor(output.outputs.probs) for output in outputs]
assert all(len(logits) == 255 for logits in vllm_logits)
for ref, ours, count in zip(ref_logits, vllm_logits, counts):
ref_probs, our_probs = option_probs(ref, count), option_probs(ours, count)
torch.testing.assert_close(our_probs, ref_probs, atol=0.02, rtol=0)
@@ -1,215 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import Qwen2VLForConditionalGeneration
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ....utils import large_gpu_test
from ...utils import check_embeddings_close
HF_TEXT_PROMPTS = [
# T -> X
(
"Query: Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501,
Image.new("RGB", (56, 56)),
),
# T -> X
(
"Query: Retrieve an image of this caption: cherry blossom",
Image.new("RGB", (56, 56)),
),
]
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "What is shown in this image?",
"cherry_blossom": "What is shown in this image?",
}
)
MODELS = ["MrLight/dse-qwen2-2b-mrl-v1"]
def get_messages(image: Image.Image, text: str, embed_text: bool):
# assert False, 'remember to use outer [] as required'
if embed_text:
messages = [
{
"role": "user",
"content": [
{
"type": "image",
"image": Image.new("RGB", (56, 56)),
"resized_height": 1,
"resized_width": 1,
}, # need a dummy image here for an easier process.
{"type": "text", "text": text},
],
}
]
else:
messages = [
{
"role": "user",
"content": [
{"type": "image", "image": image},
{"type": "text", "text": text},
],
}
]
return messages
def apply_chat_template_and_add_eos(
messages: list[dict],
apply_chat_template_fn: Callable,
):
prompt = (
apply_chat_template_fn(messages, tokenize=False, add_generation_prompt=True)
+ "<|endoftext|>"
)
return prompt
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_texts: list[str],
input_images: PromptImageInput,
embed_texts: list[bool],
model: str,
*,
dtype: str,
) -> None:
"""SET PYTHONPATH."""
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
with vllm_runner(
model, runner="pooling", dtype=dtype, enforce_eager=True, max_model_len=8192
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
texts = [
# this is necessary because vllm_model.embed will not apply any
# templating to the prompt, and therefore lacks an image_pad
# token unless one is inserted beforehand (the (28,28) image
# above is converted to an image pad token by the chat template).
apply_chat_template_and_add_eos(
get_messages(image, text, False),
apply_chat_template_fn=tokenizer.apply_chat_template,
)
for text, image in zip(input_texts, input_images)
# vllm will replace the pad token with the actual image,
# which may be a placeholder image, later.
]
vllm_outputs = vllm_model.embed(texts, images=input_images)
hf_outputs = []
with hf_runner(
model, dtype=dtype, auto_cls=Qwen2VLForConditionalGeneration
) as hf_model:
prompts = []
for text, image, embed_text in zip(input_texts, input_images, embed_texts):
# dse requires non-standard input processing
# because it needs an image_pad token
messages = get_messages(image, text, embed_text)
prompt = apply_chat_template_and_add_eos(
messages, hf_model.processor.apply_chat_template
)
prompts.append(prompt)
all_inputs = hf_model.get_inputs(
prompts=prompts,
images=input_images,
)
with torch.no_grad():
all_outputs = []
for inputs in all_inputs:
inputs = hf_model.model.prepare_inputs_for_generation(
**inputs,
cache_position=torch.arange(1), # 1 for batch size
use_cache=False,
)
outputs = hf_model.model(
**hf_model.wrap_device(inputs),
return_dict=True,
output_hidden_states=True,
)
pooled_output = F.normalize(
outputs.hidden_states[-1][0, -1], p=2, dim=-1
)
all_outputs.append(pooled_output.tolist())
hf_outputs = all_outputs
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_models_text(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [
(text, image_placeholder) for text, image_placeholder in HF_TEXT_PROMPTS
]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
embed_texts = [True] * len(input_texts)
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images, # type: ignore
embed_texts,
model,
dtype=dtype,
)
@large_gpu_test(min_gb=48)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_models_image(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
embed_texts = [False] * len(input_texts)
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images,
embed_texts,
model,
dtype=dtype,
)
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.clip import (
dual_encoder_has_text_tokens,
merge_dual_encoder_text_and_vision,
)
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
def test_text_only_batch_runs_text_encoder():
assert dual_encoder_has_text_tokens(False, None)
assert dual_encoder_has_text_tokens(False, torch.zeros(4, dtype=torch.bool))
def test_vision_only_batch_skips_text_encoder():
is_multimodal = torch.ones(8, dtype=torch.bool)
assert not dual_encoder_has_text_tokens(True, is_multimodal)
def test_mixed_batch_runs_text_encoder():
# Image tokens followed by a text sequence: the old batch-wide flag
# treated this as vision-only and skipped the text encoder (#53091).
is_multimodal = torch.tensor([True, True, True, False, False, False])
assert dual_encoder_has_text_tokens(True, is_multimodal)
def test_missing_token_mask_with_mm_embeddings_is_vision_only():
assert not dual_encoder_has_text_tokens(True, None)
def test_merge_keeps_vision_on_mm_tokens():
text = torch.zeros(4, 2)
vision = torch.ones(4, 2)
is_multimodal = torch.tensor([True, True, False, False])
out = merge_dual_encoder_text_and_vision(text, vision, is_multimodal)
assert torch.equal(out[:2], vision[:2])
assert torch.equal(out[2:], text[2:])
@@ -1,660 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import inspect
import os
import textwrap
import pytest
import torch
MODEL = os.environ.get("EG2_MODEL_PATH")
needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set")
TEXT_COS, MM_COS = 0.999, 0.999
def _cos(a, b):
return torch.nn.functional.cosine_similarity(
torch.tensor(a, dtype=torch.float32),
torch.tensor(b, dtype=torch.float32),
dim=-1,
)
def _ensure_synthetic_video(
path: str, duration: float, fps: float, width: int, height: int
) -> str:
if os.path.exists(path):
os.remove(path)
import cv2
import numpy as np
fourcc_fn = getattr(cv2, "VideoWriter_fourcc", cv2.VideoWriter.fourcc)
fourcc = fourcc_fn(*"mp4v")
num_frames = int(duration * fps)
writer = cv2.VideoWriter(path, fourcc, fps, (width, height))
y_coords, x_coords = np.mgrid[0:height, 0:width]
for i in range(num_frames):
# Spatially varying color pattern that evolves dynamically over frames
frame: np.ndarray = np.zeros((height, width, 3), dtype=np.uint8)
frame[..., 0] = ((x_coords * 255 // max(1, width)) + i * 17) % 256
frame[..., 1] = ((y_coords * 255 // max(1, height)) + i * 29) % 256
frame[..., 2] = (x_coords + y_coords + i * 13) % 256
writer.write(frame)
writer.release()
return path
@pytest.fixture(scope="module")
def st_model():
from sentence_transformers import SentenceTransformer
return SentenceTransformer(
MODEL, device="cuda", model_kwargs={"torch_dtype": torch.bfloat16}
)
@needs_model
def test_structure(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
def probe(model):
lm = model.language_model
win = [layer.self_attn.attn.impl.sliding_window for layer in lm.layers]
return win, model.config.text_config.sliding_window
win, cfg_w = vm.apply_model(probe)[0]
assert (cfg_w, cfg_w) in win # (512, 512) == W-1 with W = 513
@needs_model
@pytest.mark.parametrize(
"prompt",
[
"short query",
"task: search result | query: what is vllm",
"word " * 700, # > 512: crosses sliding window
"word " * 1500, # > 1024
],
)
def test_text_parity(vllm_runner, st_model, prompt):
ref = st_model.encode([prompt], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([prompt])
sim = _cos(out, ref).min().item()
assert sim >= TEXT_COS, (
f"Text cosine {sim} < {TEXT_COS} for prompt len {len(prompt)}"
)
@needs_model
def test_mm_recipe(vllm_runner, st_model):
from PIL import Image
img = Image.new("RGB", (224, 224), color=(255, 0, 0))
prompt = "Describe this image: <|image|>"
# SentenceTransformer multimodal encode
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Image cosine {sim} < {MM_COS}"
@needs_model
def test_natural_image_parity(vllm_runner, st_model):
from PIL import Image
img_path = "/projects/gemma4-vllm/images/cat.jpg"
if not os.path.exists(img_path):
pytest.skip(f"Test asset {img_path} not found")
img = Image.open(img_path)
prompt = "What is this image? <|image|>"
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Natural image cosine {sim} < {MM_COS}"
@needs_model
def test_audio_parity(vllm_runner, st_model):
import torchaudio
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
if not os.path.exists(audio_path):
pytest.skip(f"Test asset {audio_path} not found")
wav, sr = torchaudio.load(audio_path)
mono_wav = wav.mean(dim=0).numpy()
prompt = "Describe this audio: <|audio|>"
ref = st_model.encode(
[{"audio": audio_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"audio": (mono_wav, sr)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Audio cosine {sim} < {MM_COS}"
@needs_model
def test_mixed_parity(vllm_runner, st_model):
import torchaudio
from PIL import Image
img_path = "/projects/gemma4-vllm/images/cat.jpg"
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
if not os.path.exists(img_path) or not os.path.exists(audio_path):
pytest.skip("Test assets for mixed parity not found")
img = Image.open(img_path)
wav, sr = torchaudio.load(audio_path)
mono_wav = wav.mean(dim=0).numpy()
prompt = "Describe image <|image|> and audio <|audio|>"
ref = st_model.encode(
[{"image": img, "audio": audio_path, "text": prompt}],
normalize_embeddings=True,
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[
{
"prompt": prompt,
"multi_modal_data": {
"image": img,
"audio": (mono_wav, sr),
},
}
]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Mixed cosine {sim} < {MM_COS}"
@needs_model
def test_video_parity_uniform_overflow(vllm_runner, st_model):
from transformers.video_utils import load_video
video_path = _ensure_synthetic_video(
"/tmp/test_40s.mp4", duration=40.0, fps=10.0, width=160, height=120
)
frames, meta = load_video(video_path, backend="opencv")
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"40s video cosine {sim} < {MM_COS}"
@needs_model
def test_video_parity_portrait(vllm_runner, st_model):
from transformers.video_utils import load_video
video_path = _ensure_synthetic_video(
"/tmp/test_portrait.mp4", duration=3.0, fps=10.0, width=120, height=160
)
frames, meta = load_video(video_path, backend="opencv")
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Portrait video cosine {sim} < {MM_COS}"
@needs_model
def test_tower_skip(vllm_runner, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
def nparams(m):
return sum(p.numel() for p in m.parameters()) / 1e6
with vllm_runner(
MODEL,
runner="pooling",
max_model_len=2048,
limit_mm_per_prompt={"image": 0, "audio": 0, "video": 0},
) as vm:
p_text = vm.apply_model(nparams)[0]
assert abs(p_text - 271.00) <= 0.05, (
f"Expected 271.00M text params, got {p_text}M"
)
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
p_full = vm.apply_model(nparams)[0]
assert abs(p_full - 744.37) <= 0.05, (
f"Expected 744.37M full params, got {p_full}M"
)
def test_encoder_contract():
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration as G,
)
borrowed_methods = [
"_parse_and_validate_multimodal_inputs",
"_parse_and_validate_image_input",
"_parse_and_validate_video_input",
"_parse_and_validate_audio_input",
"_process_image_input",
"_process_video_input",
"_process_audio_input",
"embed_multimodal",
"_encoder_chunk",
]
for name in borrowed_methods:
# Check method identity (L-F6 + T-F4)
assert getattr(EmbeddingGemma2Model, name) is getattr(G, name)
# AST checks: verify no zero-arg super() calls in borrowed methods
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "super"
):
assert len(node.args) > 0, (
f"Zero-arg super() call found in borrowed method {name}"
)
# AST check: all self.<attr> non-callable accesses are assigned in __init__
init_src = inspect.getsource(EmbeddingGemma2Model.__init__)
for name in borrowed_methods:
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
attr = node.attr
assert hasattr(EmbeddingGemma2Model, attr) or attr in init_src, (
f"Attribute {attr} accessed on self in {name} is neither "
f"defined on class nor in __init__"
)
@needs_model
def test_instance_attribute_access(vllm_runner):
"""Check self.<attr> accesses on a real model instance."""
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
borrowed_methods = [
"_parse_and_validate_multimodal_inputs",
"_parse_and_validate_image_input",
"_parse_and_validate_video_input",
"_parse_and_validate_audio_input",
"_process_image_input",
"_process_video_input",
"_process_audio_input",
"embed_multimodal",
"_encoder_chunk",
]
attrs_accessed = set()
for name in borrowed_methods:
func = getattr(EmbeddingGemma2Model, name)
src = textwrap.dedent(inspect.getsource(func))
tree = ast.parse(src)
for node in ast.walk(tree):
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
attrs_accessed.add(node.attr)
def check_attrs(model):
for attr in attrs_accessed:
assert hasattr(model, attr), (
f"Real instance of EmbeddingGemma2Model is missing attribute '{attr}' "
f"accessed by borrowed method"
)
return True
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
res = vm.apply_model(check_attrs)
assert res[0] is True
@needs_model
def test_video_loader_real_video(vllm_runner, st_model):
"""Decode a real MP4 file through EmbeddingGemma2VideoBackend.load_bytes
and VideoMediaIO / MediaConnector with default num_frames=32.
"""
from vllm.multimodal.media import MediaConnector
from vllm.multimodal.media.video import VideoMediaIO
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
with open(video_path, "rb") as f:
data = f.read()
# Direct backend load
frames, meta = backend_cls.load_bytes(data)
# VideoMediaIO with default num_frames=32 (VideoMediaIO passes num_frames=32)
video_io = VideoMediaIO(image_io=None, video_backend="embedding_gemma2")
loaded = video_io.load_bytes(data)
frames_io, meta_io = loaded.media
assert len(frames_io) == len(frames)
assert meta_io.get("do_sample_frames") is False
# MediaConnector with video_processor="EmbeddingGemma2VideoProcessor"
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
fetched = connector.fetch_video(
f"file://{video_path}",
video_processor="EmbeddingGemma2VideoProcessor",
)
frames_conn, meta_conn = fetched.media
assert len(frames_conn) == 32
assert meta_conn.get("do_sample_frames") is False
prompt = "Describe this video: <|video|>"
ref = st_model.encode(
[{"video": video_path, "text": prompt}], normalize_embeddings=True
)
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[
{
"prompt": prompt,
"multi_modal_data": {"video": (frames_conn, meta_conn)},
}
]
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Real video loader cosine {sim} < {MM_COS}"
@needs_model
def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model):
"""End-to-end duplicate expansion test with low-fps clip."""
import numpy as np
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
# 4s clip at 0.5 fps = 2 unique frames
lowfps_path = "/tmp/test_lowfps_05.mp4"
_ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48)
with open(lowfps_path, "rb") as f:
data = f.read()
# Default fps=1.0 -> 4 frames expanding 2 unique frames to [0, 0, 1, 1]
frames, meta = backend_cls.load_bytes(data)
assert frames.shape[0] == 4
# Compare returned frames directly, not only indices
assert np.array_equal(frames[0], frames[1])
assert not np.array_equal(frames[0], frames[2])
assert np.array_equal(frames[2], frames[3])
prompt = "Describe this video: <|video|>"
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
out = vm.embed(
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
)
ref = st_model.encode(
[{"video": lowfps_path, "text": prompt}], normalize_embeddings=True
)
sim = _cos(out, ref).min().item()
assert sim >= MM_COS, f"Duplicate expansion cosine {sim} < {MM_COS}"
assert len(out) == 1
assert len(out[0]) == 768
def test_video_loader_indices():
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
EmbeddingGemma2VideoProcessor,
)
from transformers.video_utils import VideoMetadata
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoSourceMetadata,
VideoTargetMetadata,
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
hf_proc = EmbeddingGemma2VideoProcessor()
# Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames)
src_5s = VideoSourceMetadata(total_frames_num=120, original_fps=24.0, duration=5.0)
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
vllm_indices = backend_cls.compute_frames_index_to_sample(src_5s, tgt)
hf_meta_5s = VideoMetadata(fps=24.0, duration=5.0, total_num_frames=120)
hf_indices_5s = hf_proc.sample_frames(
hf_meta_5s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_indices == hf_indices_5s
# Case 2: clip shorter than 1s (e.g. 0.5s @ 24fps = 12 frames)
src_sub1s = VideoSourceMetadata(
total_frames_num=12, original_fps=24.0, duration=0.5
)
vllm_sub1s = backend_cls.compute_frames_index_to_sample(src_sub1s, tgt)
hf_meta_sub1s = VideoMetadata(fps=24.0, duration=0.5, total_num_frames=12)
hf_indices_sub1s = hf_proc.sample_frames(
hf_meta_sub1s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_sub1s == hf_indices_sub1s
# Case 3: clip longer than 32s (e.g. 40s @ 10fps = 400 frames)
# -> uniform overflow cap 32 frames
src_40s = VideoSourceMetadata(
total_frames_num=400, original_fps=10.0, duration=40.0
)
vllm_40s = backend_cls.compute_frames_index_to_sample(src_40s, tgt)
hf_meta_40s = VideoMetadata(fps=10.0, duration=40.0, total_num_frames=400)
hf_indices_40s = hf_proc.sample_frames(
hf_meta_40s, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_40s == hf_indices_40s
assert len(vllm_40s) == 32
# Case 4: default target fps (target.fps <= 0 defaults to 1.0)
tgt_default_fps = VideoTargetMetadata(num_frames=-1, fps=-1.0, max_duration=300)
vllm_def = backend_cls.compute_frames_index_to_sample(src_5s, tgt_default_fps)
assert vllm_def == hf_indices_5s
# Case 5 (NEW-2): 60 fps / 300 s (18000 frames) vs HF sample_frames
src_300s = VideoSourceMetadata(
total_frames_num=18000, original_fps=60.0, duration=300.0
)
v_300s = backend_cls.compute_frames_index_to_sample(
src_300s, VideoTargetMetadata(num_frames=-1, fps=60.0, max_duration=300)
)
h_300s = hf_proc.sample_frames(
VideoMetadata(fps=60.0, duration=300.0, total_num_frames=18000),
fps=60.0,
max_frames=32,
overflow_strategy="uniform",
).tolist()
assert v_300s == h_300s, f"60fps/300s mismatch: {v_300s[-3:]} vs {h_300s[-3:]}"
# Case 6 (NEW-2): 1 fps / 4 h (432000 frames @ 30 native fps) vs HF sample_frames
src_4h = VideoSourceMetadata(
total_frames_num=432000, original_fps=30.0, duration=14400.0
)
v_4h = backend_cls.compute_frames_index_to_sample(
src_4h, VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
)
h_4h = hf_proc.sample_frames(
VideoMetadata(fps=30.0, duration=14400.0, total_num_frames=432000),
fps=1.0,
max_frames=32,
overflow_strategy="uniform",
).tolist()
assert v_4h == h_4h, f"1fps/4h mismatch: {v_4h[-2:]} vs {h_4h[-2:]}"
def test_video_loader_duplicate_expansion():
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
EmbeddingGemma2VideoProcessor,
)
from transformers.video_utils import VideoMetadata
from vllm.multimodal.video import (
VIDEO_LOADER_REGISTRY,
VideoSourceMetadata,
VideoTargetMetadata,
)
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
hf_proc = EmbeddingGemma2VideoProcessor()
# 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames.
# Source has fewer frames than requested sampled frames, requiring duplicates.
src_lowfps = VideoSourceMetadata(total_frames_num=2, original_fps=0.5, duration=4.0)
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
vllm_indices = backend_cls.compute_frames_index_to_sample(src_lowfps, tgt)
hf_meta = VideoMetadata(fps=0.5, duration=4.0, total_num_frames=2)
hf_indices = hf_proc.sample_frames(
hf_meta, fps=1.0, max_frames=32, overflow_strategy="uniform"
).tolist()
assert vllm_indices == hf_indices
assert len(vllm_indices) == 4
assert vllm_indices == [0, 0, 1, 1]
def test_video_media_io_embedding_gemma2():
"""Verify VideoMediaIO with default num_frames=32 loads video bytes for
EmbeddingGemma2 without error.
"""
import os
from unittest.mock import MagicMock
from vllm.multimodal.media import MediaConnector
from vllm.multimodal.media.video import VideoMediaIO
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
if not os.path.exists(video_path):
return
with open(video_path, "rb") as f:
data = f.read()
# VideoMediaIO defaults to num_frames=32
video_io = VideoMediaIO(image_io=MagicMock(), video_backend="embedding_gemma2")
loaded = video_io.load_bytes(data)
frames, meta = loaded.media
assert len(frames) == 32
assert meta.get("do_sample_frames") is False
# MediaConnector path
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
fetched = connector.fetch_video(
f"file://{video_path}",
video_processor="EmbeddingGemma2VideoProcessor",
)
frames_conn, meta_conn = fetched.media
assert len(frames_conn) == 32
assert meta_conn.get("do_sample_frames") is False
def test_config_capping_and_explicit_limits():
"""Verify EmbeddingGemma2ModelConfig backend selection, default capping,
and preservation of explicit limits.
"""
from types import SimpleNamespace
from unittest.mock import patch
from vllm.model_executor.models.config import (
EmbeddingGemma2ModelConfig,
Gemma4Config,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
# (a) When attention backend is None, default to TRITON_ATTN,
# and default capping: max_model_len > 8192 without explicit limit capped to 8192
cfg_default = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
max_model_len=262144,
original_max_model_len=None,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default)
assert cfg_default.attention_config.backend == AttentionBackendEnum.TRITON_ATTN
assert cfg_default.model_config.max_model_len == 8192
# (b) Default capping with original_max_model_len == -1
cfg_default_minus1 = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
max_model_len=131072,
original_max_model_len=-1,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default_minus1)
assert cfg_default_minus1.model_config.max_model_len == 8192
# (c) Explicit limit 4096 preserved
cfg_explicit_4k = SimpleNamespace(
attention_config=SimpleNamespace(backend=None),
model_config=SimpleNamespace(
max_model_len=4096,
original_max_model_len=4096,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_4k)
assert cfg_explicit_4k.model_config.max_model_len == 4096
# (d) Explicit limit 16384 preserved and explicit backend preserved
cfg_explicit_16k = SimpleNamespace(
attention_config=SimpleNamespace(backend=AttentionBackendEnum.FLASH_ATTN),
model_config=SimpleNamespace(
max_model_len=16384,
original_max_model_len=16384,
),
)
with patch.object(Gemma4Config, "verify_and_update_config"):
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_16k)
assert cfg_explicit_16k.attention_config.backend == AttentionBackendEnum.FLASH_ATTN
assert cfg_explicit_16k.model_config.max_model_len == 16384
def test_sentence_transformer_tokenizer_config_max_seq_length(tmp_path):
"""Verify get_sentence_transformer_tokenizer_config accepts config
with only max_seq_length.
"""
import json
from vllm.transformers_utils.config import (
get_sentence_transformer_tokenizer_config,
)
# 1. Synthetic model directory with only max_seq_length in sentence_bert_config.json
cfg_file = tmp_path / "sentence_bert_config.json"
cfg_file.write_text(json.dumps({"max_seq_length": 8192}))
loaded = get_sentence_transformer_tokenizer_config(str(tmp_path))
assert loaded is not None
assert loaded["max_seq_length"] == 8192
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel, CLIPImageProcessor, PreTrainedModel
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from ....conftest import ImageTestAssets
# we use snapshot_download to prevent conflicts between
# dynamic_module and trust_remote_code for hf_runner
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
DEVICE_TYPE = current_platform.device_type
@torch.inference_mode()
def run_intern_vit_test(
image_assets: ImageTestAssets,
model_id: str,
*,
dtype: str,
):
model = hf_api().snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
img_processor = CLIPImageProcessor.from_pretrained(model)
images = [asset.pil_image for asset in image_assets]
pixel_values = [
img_processor(images, return_tensors="pt").pixel_values.to(torch_dtype)
for images in images
]
config = AutoConfig.from_pretrained(model, trust_remote_code=True)
if not getattr(config, "norm_type", None):
config.norm_type = "rms_norm"
# Monkey-patch for transformers v5 compatibility: InternVisionModel's
# custom code doesn't call post_init(), so all_tied_weights_keys is
# never set. Remove once https://github.com/OpenGVLab/InternVL fixes it.
_orig_init = PreTrainedModel.__init__
def _patched_init(self, *args, **kwargs):
_orig_init(self, *args, **kwargs)
self.all_tied_weights_keys = {}
PreTrainedModel.__init__ = _patched_init
try:
hf_model = AutoModel.from_pretrained(
model, dtype=torch_dtype, trust_remote_code=True
).to(DEVICE_TYPE)
finally:
PreTrainedModel.__init__ = _orig_init
hf_outputs_per_image = [
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
for pixel_value in pixel_values
]
from vllm.model_executor.models.intern_vit import InternVisionModel
vllm_model = InternVisionModel(config)
vllm_model.load_weights(hf_model.state_dict().items())
del hf_model
cleanup_dist_env_and_memory()
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
vllm_outputs_per_image = [
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
for pixel_value in pixel_values
]
del vllm_model
cleanup_dist_env_and_memory()
cos_similar = nn.CosineSimilarity(dim=-1)
for vllm_output, hf_output in zip(vllm_outputs_per_image, hf_outputs_per_image):
assert cos_similar(vllm_output, hf_output).mean() > 0.99
@pytest.mark.parametrize(
"model_id",
[
"OpenGVLab/InternViT-300M-448px",
"OpenGVLab/InternViT-6B-448px-V1-5",
],
)
@pytest.mark.parametrize("dtype", ["half"])
def test_models(
default_vllm_config, dist_init, image_assets, model_id, dtype: str
) -> None:
run_intern_vit_test(
image_assets,
model_id,
dtype=dtype,
)
@@ -1,332 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from transformers import AutoModel
from vllm.assets.base import VLLM_S3_BUCKET_URL
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageEmbedsParam,
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from ....conftest import HfRunner, VllmRunner
MODELS = ["jinaai/jina-reranker-m0"]
MM_PROCESSOR_KWARGS = {
"min_pixels": 3136,
"max_pixels": 602112,
}
LIMIT_MM_PER_PROMPT = {"image": 2}
CHECKPOINT_TO_HF_MAPPER = {
r"^visual\.": "model.visual.",
r"^model\.(?!language_model\.|visual\.)": "model.language_model.",
}
HANDELSBLATT_IMAGE_URL = (
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-handelsblatt-preview.png"
)
PAPER_IMAGE_URL = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-paper-11.png"
# Shared long text for test data
LONG_TEXT_DOC = """We present ReaderLM-v2, a compact 1.5 billion parameter language model designed for efficient
web content extraction. Our model processes documents up to 512K tokens, transforming messy HTML
into clean Markdown or JSON formats with high accuracy -- making it an ideal tool for grounding
large language models. The models effectiveness results from two key innovations: (1) a three-stage
data synthesis pipeline that generates high quality, diverse training data by iteratively drafting,
refining, and critiquing web content extraction; and (2) a unified training framework combining
continuous pre-training with multi-objective optimization. Intensive evaluation demonstrates that
ReaderLM-v2 outperforms GPT-4o-2024-08-06 and other larger models by 15-20% on carefully curated
benchmarks, particularly excelling at documents exceeding 100K tokens, while maintaining significantly
lower computational requirements.""" # noqa: E501
# Test data for different scenarios
TEXT_IMAGE_TEST_DATA = {
"query": [{"text": "slm markdown"}],
"documents": [
{"image": HANDELSBLATT_IMAGE_URL},
{"image": PAPER_IMAGE_URL},
],
}
TEXT_TEXT_TEST_DATA = {
"query": [{"text": "slm markdown"}],
"documents": [
{"text": LONG_TEXT_DOC},
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
],
}
IMAGE_TEXT_TEST_DATA = {
"query": [{"image": PAPER_IMAGE_URL}],
"documents": [
{"text": LONG_TEXT_DOC},
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
],
}
IMAGE_IMAGE_TEST_DATA = {
"query": [{"image": PAPER_IMAGE_URL}],
"documents": [
{"image": HANDELSBLATT_IMAGE_URL},
{"image": PAPER_IMAGE_URL},
],
}
TEXT_MIXED_DOCS_TEST_DATA = {
"query": [{"text": "slm markdown"}],
"documents": [
{"text": LONG_TEXT_DOC},
{"image": PAPER_IMAGE_URL},
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
{"image": HANDELSBLATT_IMAGE_URL},
],
}
def _normalize_image(image_val: str) -> str:
"""Normalize image value to proper format for HF model."""
return (
image_val
if image_val.startswith(("http://", "https://"))
else f"data:image/png;base64,{image_val}"
)
def create_score_multimodal_param(
content_parts: list[dict],
) -> list[ScoreMultiModalParam]:
"""Create a ScoreMultiModalParam from a list of content dictionaries.
Each dict supports the following formats:
- Text: {'text': 'content'}
- Image URL: {'image': 'https://...'}
- Image Base64: {'image': 'base64_str'}
"""
formatted_content = []
for part in content_parts:
if "text" in part:
formatted_content.append(
ChatCompletionContentPartTextParam(
type="text",
text=part["text"],
)
)
elif "image" in part:
image_val = part["image"]
if image_val.startswith(("http://", "https://")):
formatted_content.append(
ChatCompletionContentPartImageParam(
type="image_url",
image_url={"url": image_val},
)
)
else:
formatted_content.append(
ChatCompletionContentPartImageEmbedsParam(
type="image_embeds", image_embeds=image_val
)
)
return [ScoreMultiModalParam(content=[content]) for content in formatted_content]
def _run_vllm(
vllm_runner: type[VllmRunner],
model: str,
dtype: str,
query_strs: list[dict[str, str]],
document_strs: list[dict[str, str]],
) -> list[float]:
"""Run vLLM reranker and return scores."""
query = create_score_multimodal_param(query_strs)
documents = create_score_multimodal_param(document_strs)
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
max_num_seqs=2,
max_model_len=2048,
mm_processor_kwargs=MM_PROCESSOR_KWARGS,
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
) as vllm_model:
outputs = vllm_model.llm.score(query, documents)
return [output.outputs.score for output in outputs]
def _run_hf(
hf_runner: type[HfRunner],
model: str,
dtype: str,
query_strs: list[dict[str, str]],
document_strs: list[dict[str, str]],
) -> list[float]:
"""Run HuggingFace reranker and return scores."""
query = query_strs[0]
if "text" in query:
query_type = "text"
query_data = query["text"]
elif "image" in query:
query_type = "image"
query_data = _normalize_image(query["image"])
else:
raise ValueError("Unsupported query format")
scores: list[float] = []
with hf_runner(
model,
dtype=dtype,
trust_remote_code=True,
auto_cls=AutoModel,
model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER},
) as hf_model:
for doc in document_strs:
if "text" in doc:
score = hf_model.model.compute_score(
[[query_data, doc["text"]]],
max_length=2048,
query_type=query_type,
doc_type="text",
)
scores.append(score)
elif "image" in doc:
score = hf_model.model.compute_score(
[[query_data, doc["image"]]],
max_length=2048,
query_type=query_type,
doc_type="image",
)
scores.append(score)
return scores
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: str,
dtype: str,
query_strs: list[dict[str, str]],
document_strs: list[dict[str, str]],
) -> None:
"""Run comparison test between vLLM and HuggingFace implementations."""
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
vllm_outputs = _run_vllm(vllm_runner, model, dtype, query_strs, document_strs)
hf_outputs = _run_hf(hf_runner, model, dtype, query_strs, document_strs)
# Compare outputs
assert len(hf_outputs) == len(vllm_outputs), (
f"Output length mismatch: HF={len(hf_outputs)}, vLLM={len(vllm_outputs)}"
)
for i, (hf_score, vllm_score) in enumerate(zip(hf_outputs, vllm_outputs)):
assert hf_score == pytest.approx(vllm_score, rel=0.02), (
f"Score mismatch at index {i}: HF={hf_score}, vLLM={vllm_score}"
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_model_text_image(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
"""Visual Documents Reranking."""
_run_test(
hf_runner,
vllm_runner,
model,
dtype,
TEXT_IMAGE_TEST_DATA["query"],
TEXT_IMAGE_TEST_DATA["documents"],
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_model_text_text(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
"""Textual Documents Reranking."""
_run_test(
hf_runner,
vllm_runner,
model,
dtype,
TEXT_TEXT_TEST_DATA["query"],
TEXT_TEXT_TEST_DATA["documents"],
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_model_image_text(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
"""Image Querying for Textual Documents."""
_run_test(
hf_runner,
vllm_runner,
model,
dtype,
IMAGE_TEXT_TEST_DATA["query"],
IMAGE_TEXT_TEST_DATA["documents"],
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_model_image_image(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
"""Image Querying for Image Documents."""
_run_test(
hf_runner,
vllm_runner,
model,
dtype,
IMAGE_IMAGE_TEST_DATA["query"],
IMAGE_IMAGE_TEST_DATA["documents"],
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_model_text_mixed_documents(
hf_runner,
vllm_runner,
model: str,
dtype: str,
) -> None:
"""Text Query for Mixed Text and Image Documents."""
_run_test(
hf_runner,
vllm_runner,
model,
dtype,
TEXT_MIXED_DOCS_TEST_DATA["query"],
TEXT_MIXED_DOCS_TEST_DATA["documents"],
)
@@ -1,394 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the LlamaNemotronVL model family:
- nvidia/llama-nemotron-embed-vl-1b-v2 (LlamaNemotronVLForCausalLM / embed)
- nvidia/llama-nemotron-rerank-vl-1b-v2
(LlamaNemotronVLForSequenceClassification / rerank)
Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
"""
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
import pybase64 as base64
import pytest
import torch
from PIL import Image
from transformers import AutoModel, AutoModelForSequenceClassification, AutoProcessor
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
LlamaNemotronVLEmbedImageProcessor,
)
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ....utils import ROCM_ENGINE_KWARGS
from ...utils import check_embeddings_close
# Prefixes used by the model API
QUERY_PREFIX = "query: "
PASSAGE_PREFIX = "passage: "
# Text prompts for text-only embedding
HF_TEXT_PROMPTS = [
# T -> X (text embedding queries)
f"{QUERY_PREFIX}The label of the object is stop sign",
f"{QUERY_PREFIX}cherry blossom",
]
# Image prompts using the model's expected format
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
# I -> X (image embedding as passage/document)
"stop_sign": f"{PASSAGE_PREFIX}<image>",
"cherry_blossom": f"{PASSAGE_PREFIX}<image>",
}
)
MODELS = ["nvidia/llama-nemotron-embed-vl-1b-v2"]
@pytest.mark.usefixtures("default_vllm_config")
def test_device_normalization_matches_cpu_transform() -> None:
device = current_platform.device_type
image = Image.new("RGB", (48, 32), color=(12, 128, 240))
processor_kwargs = dict(
image_size=32,
min_dynamic_patch=1,
max_dynamic_patch=1,
dynamic_image_size=True,
use_thumbnail=False,
)
cpu_processor = LlamaNemotronVLEmbedImageProcessor(**processor_kwargs)
device_processor = LlamaNemotronVLEmbedImageProcessor(
**processor_kwargs, do_rescale=False, do_normalize=False
)
expected = cpu_processor._images_to_pixel_values_lst([image])[0]
raw_pixels = device_processor._images_to_pixel_values_lst([image])[0]
input_norm = FusedMMInputNorm(
image_mean=list(SIGLIP_MEAN),
image_std=list(SIGLIP_STD),
rescale_factor=1 / 255,
).to(device)
actual = input_norm(
raw_pixels.flatten(start_dim=1).to(device), expected.dtype
).view_as(expected)
assert raw_pixels.dtype == torch.uint8
torch.testing.assert_close(actual, expected.to(device))
@pytest.mark.parametrize(("do_rescale", "do_normalize"), [(True, False), (False, True)])
def test_device_normalization_rejects_partial_cpu_processing(
do_rescale: bool, do_normalize: bool
) -> None:
with pytest.raises(ValueError, match="both enabled or both disabled"):
LlamaNemotronVLEmbedImageProcessor(
image_size=32,
min_dynamic_patch=1,
max_dynamic_patch=1,
dynamic_image_size=True,
use_thumbnail=False,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_cases: list[tuple[list[str], PromptImageInput]],
model: str,
*,
dtype: str,
) -> None:
"""Compare HF and vLLM embeddings for all input cases.
NOTE: Run vLLM first to avoid CUDA initialization issues with multiprocessing.
"""
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
max_model_len=2048,
enforce_eager=True,
trust_remote_code=True,
**ROCM_ENGINE_KWARGS,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.embed(input_texts, images=input_images)
for input_texts, input_images in input_cases
]
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_outputs_per_case = []
for input_texts, input_images in input_cases:
hf_outputs = []
for text, image in zip(input_texts, input_images):
with torch.inference_mode():
if text.startswith(QUERY_PREFIX):
query_text = text[len(QUERY_PREFIX) :]
embedding = hf_model.model.encode_queries([query_text])
elif text.startswith(PASSAGE_PREFIX):
passage_text = text[len(PASSAGE_PREFIX) :]
if image is not None:
embedding = hf_model.model.encode_documents(
images=[image],
texts=[passage_text],
)
else:
embedding = hf_model.model.encode_documents(
texts=[passage_text]
)
else:
raise ValueError(
f"Text must start with {QUERY_PREFIX!r} "
f"or {PASSAGE_PREFIX!r}"
)
hf_outputs.append(embedding[0].tolist())
hf_outputs_per_case.append(hf_outputs)
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
"""Test text and image embedding."""
text_images = [None] * len(HF_TEXT_PROMPTS)
images = [asset.pil_image for asset in image_assets]
input_cases = [
(HF_TEXT_PROMPTS, text_images),
(HF_IMAGE_PROMPTS, images),
]
_run_test(
hf_runner,
vllm_runner,
input_cases,
model,
dtype=dtype,
)
# ---------------------------------------------------------------------------
# Reranker tests — nvidia/llama-nemotron-rerank-vl-1b-v2
# ---------------------------------------------------------------------------
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
# APIs (it's inherited from the base LLM). We must use the provided override.
_RERANKER_SCORE_TEMPLATE = (
Path(__file__).parents[4]
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
).read_text()
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
RERANKER_TEXT_DOCS = [
"AI enables robots to perceive, plan, and act autonomously.",
(
"A biological foundation model designed to analyze DNA, RNA, "
"and protein sequences."
),
]
RERANKER_IMAGE_QUERY = "photo of a red stop sign on a street"
RerankerDocument = tuple[str | None, Image.Image | None]
RerankerCase = tuple[str, Sequence[RerankerDocument]]
def _pil_to_data_uri(image: Image.Image) -> str:
buf = BytesIO()
image.save(buf, format="PNG")
b64 = base64.b64encode(buf.getvalue()).decode()
return f"data:image/png;base64,{b64}"
def _run_hf_reranker(
hf_runner: type[HfRunner],
model: str,
dtype: str,
input_cases: Sequence[RerankerCase],
) -> list[list[float]]:
"""Run all HF reranker cases in one model lifecycle."""
with hf_runner(
model,
dtype=dtype,
trust_remote_code=True,
auto_cls=AutoModelForSequenceClassification,
) as hf_model:
processor = AutoProcessor.from_pretrained(
model,
trust_remote_code=True,
max_input_tiles=6,
use_thumbnail=True,
rerank_max_length=2048,
)
scores_per_case = []
for query, docs in input_cases:
examples = [
{
"question": query,
"doc_text": doc_text if doc_text is not None else "",
"doc_image": doc_image if doc_image is not None else "",
}
for doc_text, doc_image in docs
]
batch_dict = processor.process_queries_documents_crossencoder(examples)
batch_dict = {
k: v.to(hf_model.model.device) if isinstance(v, torch.Tensor) else v
for k, v in batch_dict.items()
}
with torch.inference_mode():
logits = hf_model.model(**batch_dict, return_dict=True).logits
# vLLM applies sigmoid activation to raw logits before returning scores.
scores = torch.sigmoid(logits.squeeze(-1).float())
scores_per_case.append(scores.detach().cpu().tolist())
return scores_per_case
def _run_vllm_reranker(
vllm_runner: type[VllmRunner],
model: str,
dtype: str,
input_cases: Sequence[RerankerCase],
) -> list[list[float]]:
"""Run all vLLM reranker cases in one model lifecycle."""
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
max_model_len=2048,
enforce_eager=True,
trust_remote_code=True,
**ROCM_ENGINE_KWARGS,
) as vllm_model:
scores_per_case = []
for query, docs in input_cases:
has_images = any(img is not None for _, img in docs)
if not has_images:
queries = [query] * len(docs)
doc_texts = [doc_text for doc_text, _ in docs]
outputs = vllm_model.score(
queries,
doc_texts,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
else:
query_params = [
ScoreMultiModalParam(
content=[
ChatCompletionContentPartTextParam(
type="text",
text=query,
)
]
)
] * len(docs)
doc_params = []
for doc_text, doc_image in docs:
content: list = []
if doc_image is not None:
content.append(
ChatCompletionContentPartImageParam(
type="image_url",
image_url={"url": _pil_to_data_uri(doc_image)},
)
)
if doc_text:
content.append(
ChatCompletionContentPartTextParam(
type="text",
text=doc_text,
)
)
doc_params.append(ScoreMultiModalParam(content=content))
raw_outputs = vllm_model.llm.score(
query_params,
doc_params,
chat_template=_RERANKER_SCORE_TEMPLATE,
)
outputs = [output.outputs.score for output in raw_outputs]
scores_per_case.append(outputs)
return scores_per_case
def _run_reranker_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
model: str,
dtype: str,
input_cases: Sequence[RerankerCase],
) -> None:
"""Compare HF and vLLM reranker scores for all input cases.
NOTE: Run vLLM first to avoid CUDA initialization issues with multiprocessing.
"""
vllm_scores_per_case = _run_vllm_reranker(vllm_runner, model, dtype, input_cases)
hf_scores_per_case = _run_hf_reranker(hf_runner, model, dtype, input_cases)
# ROCm has slightly higher variance because vLLM and HF use different
# attention backends.
rel_tol = 0.022 if current_platform.is_rocm() else 0.02
for hf_scores, vllm_scores in zip(hf_scores_per_case, vllm_scores_per_case):
assert len(hf_scores) == len(vllm_scores), (
f"Output length mismatch: HF={len(hf_scores)}, vLLM={len(vllm_scores)}"
)
for i, (hf_score, vllm_score) in enumerate(zip(hf_scores, vllm_scores)):
assert hf_score == pytest.approx(vllm_score, rel=rel_tol), (
f"Score mismatch at index {i}: HF={hf_score:.4f}, vLLM={vllm_score:.4f}"
)
@pytest.mark.parametrize("model", RERANKER_MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_reranker(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
"""Test reranking with text and image documents."""
text_docs: list[RerankerDocument] = [(text, None) for text in RERANKER_TEXT_DOCS]
image_docs: list[RerankerDocument] = [
(None, asset.pil_image) for asset in image_assets
]
input_cases: list[RerankerCase] = [
(RERANKER_TEXT_QUERY, text_docs),
(RERANKER_IMAGE_QUERY, image_docs),
]
_run_reranker_test(hf_runner, vllm_runner, model, dtype, input_cases)
@@ -1,159 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch.nn.functional as F
from transformers import AutoModelForImageTextToText
from vllm.platforms import current_platform
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ....utils import large_gpu_test
from ...utils import check_embeddings_close
# Llava Next embedding implementation is only supported by CUDA.
# If run on ROCm, hf_model.model.resize_token_embeddings will
# cause the following error:
# RuntimeError: Calling torch.linalg.cholesky on a CUDA tensor
# requires compiling PyTorch with MAGMA. Please use PyTorch
# built with MAGMA support.
# If run on CPU, hf_model.model.resize_token_embeddings will
# cause the following error:
# RuntimeError: Calling torch.linalg.cholesky on a CPU tensor
# requires compiling PyTorch with LAPACK. Please use PyTorch
# built with LAPACK support.
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Llava Next model uses op that is only supported in CUDA",
)
llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501
HF_TEXT_PROMPTS = [
# T -> X
llama3_template.format(
"The label of the object is stop sign\nSummary above sentence in one word: " # noqa: E501
),
# T -> X
llama3_template.format("cherry blossom\nSummary above sentence in one word: "),
]
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
# I -> X
"stop_sign": llama3_template.format(
"<image>\nSummary above image in one word: "
),
# I -> X
"cherry_blossom": llama3_template.format(
"<image>\nSummary above image in one word: "
),
}
)
MODELS = ["royokong/e5-v"]
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_texts: list[str],
input_images: PromptImageInput,
model: str,
*,
dtype: str,
) -> None:
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
with vllm_runner(
model, runner="pooling", dtype=dtype, max_model_len=4096, enforce_eager=True
) as vllm_model:
vllm_outputs = vllm_model.embed(input_texts, images=input_images)
with hf_runner(
model, dtype=dtype, auto_cls=AutoModelForImageTextToText
) as hf_model:
# Patch the issue where generation_config.json is missing
hf_model.processor.patch_size = hf_model.model.config.vision_config.patch_size
# Patch the issue where image_token_id
# exceeds the maximum allowed vocab size
hf_model.model.resize_token_embeddings(
hf_model.model.model.language_model.vocab_size + 1
)
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
all_outputs = []
for inputs in all_inputs:
# Based on: https://huggingface.co/royokong/e5-v
outputs = hf_model.model(
**hf_model.wrap_device(inputs),
return_dict=True,
output_hidden_states=True,
)
pooled_output = F.normalize(outputs.hidden_states[-1][0, -1, :], dim=-1)
all_outputs.append(pooled_output.tolist())
hf_outputs = all_outputs
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.core_model
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models_text(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images, # type: ignore
model,
dtype=dtype,
)
@large_gpu_test(min_gb=48)
@pytest.mark.core_model
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models_image(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images,
model,
dtype=dtype,
)
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn.functional as F
import transformers.utils
from PIL import Image
from vllm.assets.base import get_vllm_public_assets
from vllm.assets.image import VLM_IMAGES_DIR
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ....utils import large_gpu_test
from ...utils import check_embeddings_close
# BC for method that was deleted in Transformers v5.
# Only needed for generating the HF reference.
transformers.utils.is_flash_attn_greater_or_equal_2_10 = (
lambda: transformers.utils.is_flash_attn_greater_or_equal("2.1.0")
)
HF_TEXT_PROMPTS = [
# T -> X
"Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501
# T -> X
"Retrieve an image of this caption: cherry blossom",
]
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
# T + I -> X
"stop_sign": "<|image_1|> Select the portion of the image that isolates the object of the given label: The label of the object is stop sign", # noqa: E501
# I -> X
"cherry_blossom": "<|image_1|> Represent the given image for classification", # noqa: E501
}
)
MODELS = ["TIGER-Lab/VLM2Vec-Full"]
SPECIAL_TOKEN_IMAGE_PROMPT = (
"\n<s><|user|>\n <|image_1|>\n\t <s>"
"Represent the given image for classification<|end|>"
"\n<|assistant|>\n"
)
def _get_cherry_blossom_image() -> Image.Image:
return Image.open(
get_vllm_public_assets(filename="cherry_blossom.jpg", s3_prefix=VLM_IMAGES_DIR)
)
@torch.inference_mode()
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_texts: list[str],
input_images: PromptImageInput,
model: str,
*,
dtype: str,
) -> None:
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
with vllm_runner(
model, runner="pooling", dtype=dtype, enforce_eager=True
) as vllm_model:
vllm_outputs = vllm_model.embed(input_texts, images=input_images)
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
hf_model_kwargs = {"_attn_implementation": "eager"}
with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
all_outputs = []
for inputs in all_inputs:
# Based on: https://github.com/TIGER-AI-Lab/VLM2Vec/blob/db3b951bccabba220c1f53ab46a734e50dd2fc08/src/model.py
outputs = hf_model.model(
**hf_model.wrap_device(inputs),
return_dict=True,
output_hidden_states=True,
)
last_hidden_state = outputs.hidden_states[-1][0]
reps = last_hidden_state[inputs.attention_mask[0].sum() - 1]
pooled_output = F.normalize(reps, p=2, dim=-1)
all_outputs.append(pooled_output.tolist())
hf_outputs = all_outputs
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.core_model
@pytest.mark.skipif(
current_platform.is_xpu(),
reason=(
"vllm-xpu-kernels paged_decode_default.conf is missing the "
"8,96,64,false,true,false tuple needed by VLM2Vec-Full at "
"block_size=64. Re-enable once vllm-xpu-kernels ships the updated "
"config. See github.com/vllm-project/vllm-xpu-kernels/pull/587."
),
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models_text(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images, # type: ignore
model,
dtype=dtype,
)
@large_gpu_test(min_gb=48)
@pytest.mark.core_model
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models_image(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
input_texts_images = [
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
]
input_texts = [text for text, _ in input_texts_images]
input_images = [image for _, image in input_texts_images]
_run_test(
hf_runner,
vllm_runner,
input_texts,
input_images,
model,
dtype=dtype,
)
@pytest.mark.core_model
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half"])
def test_models_image_special_tokens_processing(
model: str,
dtype: str,
) -> None:
model_config = ModelConfig(
model,
runner="pooling",
trust_remote_code=True,
dtype=dtype,
max_model_len=1024,
)
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
image = _get_cherry_blossom_image()
processed_inputs = processor(
SPECIAL_TOKEN_IMAGE_PROMPT,
mm_items=processor.info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
hf_processor = processor.info.get_hf_processor()
hf_inputs = hf_processor(
SPECIAL_TOKEN_IMAGE_PROMPT,
images=image,
return_tensors="pt",
)
image_token_id = hf_processor.get_special_image_token_id()
hf_prompt_token_ids = [
image_token_id if token_id < 0 else token_id
for token_id in hf_inputs["input_ids"][0].tolist()
]
prompt_token_ids = processed_inputs["prompt_token_ids"]
assert prompt_token_ids == hf_prompt_token_ids
assert prompt_token_ids.count(image_token_id) == hf_prompt_token_ids.count(
image_token_id
)
assert prompt_token_ids.count(image_token_id) > 0
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import pytest
import torch
from ....conftest import VllmRunner
pytestmark = pytest.mark.skipif(
importlib.util.find_spec("terratorch") is None,
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
)
def _run_test(
vllm_runner: type[VllmRunner],
model: str,
) -> None:
prompt = [
{
# This model deals with no text input
"prompt_token_ids": [1],
"multi_modal_data": {
"image": {
"pixel_values": torch.ones((6, 512, 512), dtype=torch.float16),
"location_coords": torch.ones((1, 2), dtype=torch.float16),
}
},
}
for _ in range(10)
]
with vllm_runner(
model,
runner="pooling",
dtype="half",
enforce_eager=True,
skip_tokenizer_init=True,
enable_mm_embeds=True,
# Limit the maximum number of sequences to avoid the
# test going OOM during the warmup run
max_num_seqs=32,
default_torch_num_threads=1,
) as vllm_model:
vllm_model.llm.encode(prompt, pooling_task="plugin")
MODELS = ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"]
@pytest.mark.core_model
@pytest.mark.parametrize("model", MODELS)
def test_models_image(
hf_runner,
vllm_runner,
image_assets,
model: str,
) -> None:
_run_test(
vllm_runner,
model,
)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
MODEL = "Qwen/Qwen3-ForcedAligner-0.6B"
CLASSIFY_NUM = 5000
TIMESTAMP_TOKEN_ID = 151705
def build_prompt(words: list[str]) -> str:
body = "<timestamp><timestamp>".join(words) + "<timestamp><timestamp>"
return f"<|audio_start|><|audio_pad|><|audio_end|>{body}"
@pytest.mark.parametrize("model", [MODEL])
@pytest.mark.parametrize("dtype", ["bfloat16"])
@torch.inference_mode()
def test_qwen3_forced_aligner(
vllm_runner,
model: str,
dtype: str,
) -> None:
words = ["Hello", "world"]
prompt = build_prompt(words)
# 5-second silent audio at 16kHz
audio = np.zeros(16000 * 5, dtype=np.float32)
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
enforce_eager=True,
max_model_len=512,
hf_overrides={
"architectures": [
"Qwen3ASRForcedAlignerForTokenClassification",
],
},
) as vllm_model:
outputs = vllm_model.llm.encode(
[{"prompt": prompt, "multi_modal_data": {"audio": audio}}],
pooling_task="token_classify",
)
# Validate output structure
assert len(outputs) == 1
logits = outputs[0].outputs.data
assert logits.dim() == 2
assert logits.shape[1] == CLASSIFY_NUM
# Validate timestamp extraction
token_ids = outputs[0].prompt_token_ids
predictions = logits.argmax(dim=-1)
ts_indices = [i for i, t in enumerate(token_ids) if t == TIMESTAMP_TOKEN_ID]
# 2 words x 2 timestamps each (start + end) = 4
assert len(ts_indices) == 4
ts_preds = [predictions[i].item() for i in ts_indices]
assert all(p >= 0 for p in ts_preds)
# end >= start for each word
assert ts_preds[1] >= ts_preds[0] # Hello
assert ts_preds[3] >= ts_preds[2] # world
@@ -1,106 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.models.radio import RadioModel
from vllm.platforms import current_platform
from vllm.transformers_utils.configs.radio import RadioConfig
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from ....conftest import ImageTestAssets
# we use snapshot_download to prevent conflicts between
# dynamic_module and trust_remote_code for hf_runner
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
DEVICE_TYPE = current_platform.device_type
@torch.inference_mode()
def run_radio_test(
image_assets: ImageTestAssets,
model_id: str,
*,
dtype: str,
):
model = hf_api().snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
img_processor = CLIPImageProcessor.from_pretrained(model)
images = [asset.pil_image for asset in image_assets]
# Input resolution must be a multiple of `self.min_resolution_step`.
# Using `self.get_nearest_supported_resolution`, for assets 432x642 the
# nearest supported resolution is 432x640.
pixel_values = [
img_processor(image, return_tensors="pt").pixel_values.to(torch_dtype)[
:, :, :, :640
]
for image in images
]
hf_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
# RADIO model on HF does not properly handle torch_dtype argument
# And relies on args["dtype"] which we have to patch manually:
hf_config.args["dtype"] = torch_dtype
hf_model = AutoModel.from_pretrained(
model_id,
config=hf_config,
dtype=torch_dtype,
trust_remote_code=True,
).to(DEVICE_TYPE)
hf_model.eval()
# A HF model has image normalization as a part of model's forward
# However in vLLM we don't make normalization a part of the model
# forward step since mean/std stored as model's parameters and
# subject to precision loss (when using fp16/bf16) which negatively
# affects evaluation benchmarks.
hf_model.make_preprocessor_external()
hf_outputs_per_image = [
hf_model(pixel_value.to(DEVICE_TYPE)) for pixel_value in pixel_values
]
vllm_config = RadioConfig(
model_name=hf_config.args["model"],
**hf_config.args,
)
vllm_model = RadioModel(vllm_config)
vllm_model.load_weights(hf_model.state_dict())
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
vllm_outputs_per_image = [
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
for pixel_value in pixel_values
]
del vllm_model, hf_model
cleanup_dist_env_and_memory()
cos_similar = nn.CosineSimilarity(dim=-1)
for vllm_output, hf_output in zip(vllm_outputs_per_image, hf_outputs_per_image):
assert cos_similar(vllm_output[0], hf_output[0]).mean() > 0.99
assert cos_similar(vllm_output[1], hf_output[1]).mean() > 0.99
@pytest.mark.parametrize(
"model_id",
[
"nvidia/C-RADIOv2-H",
],
)
@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
def test_radio(
default_vllm_config, dist_init, image_assets, model_id, dtype: str
) -> None:
run_radio_test(
image_assets,
model_id,
dtype=dtype,
)
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
import torch
from transformers import SiglipModel
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ...utils import check_embeddings_close
HF_TEXT_PROMPTS = [
"a photo of a stop sign",
"a photo of a cherry blossom",
]
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "",
"cherry_blossom": "",
}
)
MODELS = [
"google/siglip-base-patch16-224",
"google/siglip2-base-patch16-224",
# Different image embedding dim than text_config.hidden_size
"google/siglip2-giant-opt-patch16-384",
]
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
input_cases: list[tuple[list[str], PromptImageInput, dict[str, Any]]],
model: str,
*,
dtype: str,
) -> None:
with vllm_runner(
model,
runner="pooling",
dtype=dtype,
enforce_eager=True,
max_model_len=64,
gpu_memory_utilization=0.7,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.embed(
input_texts,
images=input_images,
tokenization_kwargs=tokenization_kwargs,
)
for input_texts, input_images, tokenization_kwargs in input_cases
]
texts = [HF_TEXT_PROMPTS[0]]
images = [input_cases[1][1][0]]
with pytest.raises(ValueError, match="not both"):
vllm_model.embed(texts, images=images)
vllm_model.embed(texts)
vllm_model.embed([""], images=images)
mixed_outputs = vllm_outputs_per_case[2]
check_embeddings_close(
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
embeddings_1_lst=[mixed_outputs[0]],
name_0="text_only",
name_1="mixed_text",
)
check_embeddings_close(
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
embeddings_1_lst=[mixed_outputs[1]],
name_0="image_only",
name_1="mixed_image",
)
with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:
hf_outputs_per_case = []
for input_texts, input_images, tokenization_kwargs in input_cases:
all_inputs = hf_model.get_inputs(
input_texts,
images=input_images,
tokenization_kwargs=tokenization_kwargs,
)
hf_outputs = []
for inputs in all_inputs:
inputs = hf_model.wrap_device(inputs)
if "pixel_values" in inputs:
pooled_output = hf_model.model.get_image_features(
pixel_values=inputs.pixel_values,
)
else:
pooled_output = hf_model.model.get_text_features(
input_ids=inputs.input_ids,
)
if not isinstance(pooled_output, torch.Tensor):
pooled_output = pooled_output.pooler_output
pooled_output = pooled_output.squeeze(0)
hf_outputs.append(pooled_output.tolist())
hf_outputs_per_case.append(hf_outputs)
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["float"])
def test_models(
hf_runner,
vllm_runner,
image_assets,
model: str,
dtype: str,
) -> None:
text_images = [None] * len(HF_TEXT_PROMPTS)
images = [asset.pil_image for asset in image_assets]
input_cases = [
(
HF_TEXT_PROMPTS,
text_images,
{
"padding": "max_length",
"max_length": 64,
},
),
(HF_IMAGE_PROMPTS, images, {}),
(
[HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]],
[None, images[0]],
{
"padding": "max_length",
"max_length": 64,
},
),
]
_run_test(
hf_runner,
vllm_runner,
input_cases,
model,
dtype=dtype,
)
@@ -1,154 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The vLLM team.
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
# reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from transformers import PreTrainedConfig
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.config.multimodal import MultiModalDummyOptions
class MockAudioFlamingo3Config(PreTrainedConfig):
model_type = "audioflamingo3"
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.audio_config = PreTrainedConfig()
self.text_config = PreTrainedConfig()
class MockAudioFlamingo3Processor:
def __init__(self):
self.audio_token = "<sound>"
self.audio_token_id = 12345
self.max_audio_len = 60
self.feature_extractor = MockFeatureExtractor()
self.tokenizer = self._tokenize
def __call__(self, text=None, audio=None, **kwargs):
return {
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
"input_features": torch.zeros((3, 80, 3000)),
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
}
def _tokenize(self, text, **kwargs):
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
class MockFeatureExtractor:
def __init__(self):
self.sampling_rate = 16000
self.chunk_length = 30
self.hop_length = 160
def __call__(self, audios, **kwargs):
return {
"input_features": torch.zeros((len(audios), 80, 3000)),
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
}
@pytest.fixture
def mock_ctx():
config = MockAudioFlamingo3Config()
ctx = MagicMock()
ctx.get_hf_config.return_value = config
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
**data, **kwargs
)
ctx.model_config.hf_config = config
return ctx
@pytest.fixture(autouse=True)
def check_transformers_version():
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
model_info.check_transformers_version(on_fail="skip")
def test_audio_chunk_counting(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3MultiModalProcessor,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
processor = AudioFlamingo3MultiModalProcessor(
info, AudioFlamingo3DummyInputsBuilder(info)
)
sr = 16000
audio_1 = np.zeros(30 * sr)
audio_2 = np.zeros(75 * sr)
mm_data = {"audio": [audio_1, audio_2]}
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
processed = processor._apply_hf_processor_main(mm_items, {})
chunk_counts = processed["chunk_counts"]
assert chunk_counts[0].item() == 1
assert chunk_counts[1].item() == 2
assert len(chunk_counts) == 2
assert processed["feature_attention_mask"].shape == (3, 3000)
def test_dummy_data_generation(mock_ctx):
from vllm.model_executor.models.audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3ProcessingInfo,
)
info = AudioFlamingo3ProcessingInfo(mock_ctx)
builder = AudioFlamingo3DummyInputsBuilder(info)
mm_counts = {"audio": 2}
dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())
assert "audio" in dummy_data
assert len(dummy_data["audio"]) == 2
expected_len = 60 * 16000
assert len(dummy_data["audio"][0]) == expected_len
def test_audio_token_count_matches_hf_processor_math():
from vllm.model_executor.models.audioflamingo3 import (
_count_audio_tokens_from_mask,
)
feature_attention_mask = torch.zeros((3, 3000), dtype=torch.long)
feature_attention_mask[0, :2999] = 1
feature_attention_mask[1, :2999] = 1
feature_attention_mask[2, :1500] = 1
chunk_counts = torch.tensor([2, 1], dtype=torch.long)
assert (
_count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 0) == 1499
)
assert _count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 1) == 375
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bailing prompt expansion tests without downloading a checkpoint."""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
from transformers import TokenizersBackend
from vllm.model_executor.models.bailing_moe_v3_vl import (
IMAGE_PLACEHOLDER,
IMAGE_TOKEN,
BailingMoeV3VLMultiModalProcessor,
)
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def processor():
vocab = {
token: index
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
}
# A newline added by the processor must merge with an existing newline.
vocab["ĊĊ"] = len(vocab)
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
add_prefix_space=False, use_regex=False
)
backend.decoder = decoders.ByteLevel()
tokenizer = TokenizersBackend(
tokenizer_object=backend,
additional_special_tokens=[
"<|vision_start|>",
IMAGE_TOKEN,
"<|vision_end|>",
],
clean_up_tokenization_spaces=False,
)
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
processor.info = SimpleNamespace(
get_tokenizer=lambda: tokenizer,
get_hf_config=lambda: SimpleNamespace(
vision_config=SimpleNamespace(spatial_merge_size=2)
),
)
return processor
@pytest.mark.parametrize(
("prompt", "expected_text", "token_counts"),
[
(
f"before{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_TOKEN}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}\nafter",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
[2],
),
(
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
[2, 3],
),
],
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
)
def test_image_expansion_matches_wrappers_and_newline_tokenization(
processor, prompt, expected_text, token_counts
):
tokenizer = processor.info.get_tokenizer()
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
{"image_grid_thw": grids},
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
)
mm_items = MultiModalDataItems(
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
)
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
assert actual_ids == expected_ids
assert len(placeholders["image"]) == len(token_counts)
found = processor._find_mm_placeholders(actual_ids, updates)
search_offset = 0
for index, count in enumerate(token_counts):
placeholder = placeholders["image"][index]
expected_offset = expected_ids.index(image_id, search_offset)
assert placeholder.start_idx == expected_offset
assert placeholder.tokens == [image_id] * count
assert placeholder.to_range().get_num_embeds() == count
assert placeholder.to_range() == found["image"][index].to_range()
search_offset = expected_offset + count
def test_text_only_prompt_preserves_original_token_ids(processor):
tokenizer = processor.info.get_tokenizer()
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
# Deliberately use two newline tokens instead of their merged BPE token.
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
assert actual_ids == prompt_ids
assert placeholders == {}
@@ -1,244 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
images that are small enough to not require cropping produce an empty
images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input
method must correctly read image_size from this tensor's shape rather than
falling back to base_size, which would cause a TensorSchema mismatch.
Run with:
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
"""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from transformers import AutoTokenizer
from vllm.model_executor.models.deepseek_ocr import (
DeepseekOCRForCausalLM,
DeepseekOCRImagePixelInputs,
)
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
from vllm.transformers_utils.processors.deepseek_ocr import (
DeepseekOCRProcessor,
ImageTransform,
)
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
@pytest.fixture(scope="module")
def processor():
"""Load the DeepseekOCRProcessor with tokenizer from HuggingFace."""
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
return DeepseekOCRProcessor(tokenizer=tokenizer)
class TestDeepseekOCREmptyImagesCrop:
"""Verify TensorSchema validation handles empty images_crop correctly."""
def test_empty_images_crop_small_image(self, processor):
"""A small image (<=640px) produces empty images_crop and should
not crash the TensorSchema validation.
Previously, the code used ``numel() > 0`` to decide whether to read
image_size from the tensor shape. When numel()==0, it fell back to
base_size=1024, mismatching the actual tensor dim of 640.
"""
# Small image: both dims <= IMAGE_SIZE (640) → no crops
small_image = Image.new("RGB", (100, 100), color="red")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
# Processor must produce an empty crop tensor for a small image
assert images_crop.shape[0] == 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1] if images_crop is not None else base_size
# This should NOT raise ValueError
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape == (0, 3, 640, 640)
def test_populated_images_crop_large_image(self, processor):
"""A large image (>640px) produces populated images_crop."""
# Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles
large_image = Image.new("RGB", (1200, 800), color="blue")
result = processor(
prompt="<image>\nDescribe this image.",
images=[large_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
assert images_crop.shape[0] > 0
base_size = pixel_values.shape[-1]
image_size = images_crop.shape[-1]
schema = DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": base_size,
"image_size": image_size,
},
)
assert schema.data.shape == (1, 3, 1024, 1024)
assert schema.images_crop.shape[-1] == 640
def test_mismatched_image_size_raises(self, processor):
"""Deliberately wrong image_size binding should still be caught
by TensorSchema validation."""
small_image = Image.new("RGB", (100, 100), color="green")
result = processor(
prompt="<image>\nDescribe this image.",
images=[small_image],
)
pixel_values = result["pixel_values"]
images_crop = result["images_crop"]
images_spatial_crop = result["images_spatial_crop"]
with pytest.raises(ValueError, match="images_crop"):
DeepseekOCRImagePixelInputs(
type="pixel_values",
data=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
resolve_bindings={
"base_size": 1024,
"image_size": 1024, # Wrong! Tensor has 640
},
)
class TestDeepseekOCRInputValidation:
"""Bare ``assert`` used to validate user input here returned HTTP 500
(AssertionError) instead of HTTP 400 (ValueError) and disappeared under
``python -O``, silently corrupting the tokenized sequence. See #53850.
"""
def test_mismatched_image_count_raises_value_error(self, processor):
"""Two ``<image>`` tokens with one image must raise ValueError, not
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
drop the middle text segment)."""
image = Image.new("RGB", (100, 100), color="red")
with pytest.raises(ValueError, match="does not match number of images"):
processor(prompt="<image> and <image>", images=[image])
def test_missing_prompt_raises_value_error(self, processor):
image = Image.new("RGB", (100, 100), color="red")
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt=None, images=[image])
def test_missing_images_raises_value_error(self, processor):
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt="<image>\nDescribe this image.", images=None)
def _balanced_normalized_pixels(
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
) -> torch.Tensor:
"""Build a pixel tensor whose values cancel to an exact sum of 0.
Matches the normalized half-black / half-white case from the DeepSeek-OCR
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
"""
assert width % 2 == 0
pixel_values = torch.ones(batch, channels, height, width)
pixel_values[..., : width // 2] = -1.0
assert pixel_values.sum().item() == 0.0
return pixel_values
class TestDeepseekOCRZeroSumPixelsAccepted:
"""Zero-sum normalized pixels must still produce validated image inputs.
Returning None from ``_parse_and_validate_image_input`` makes
``embed_multimodal`` return None and trips the worker encoder-output
sanity check, killing EngineCore. The zero-sum shortcut is therefore
removed; balanced images must parse like any other valid tensor.
"""
def test_ocr_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_ocr2_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
model,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_image_transform_half_black_half_white_sums_to_zero(self):
"""A 1024×1024 half-black/half-white PNG through ImageTransform
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
must still accept it.
"""
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
pixel_values = ImageTransform()(image).unsqueeze(0)
assert pixel_values.sum().item() == 0.0
images_crop = torch.zeros(0, 3, 640, 640)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert torch.equal(image_input.data, pixel_values)
@@ -1,67 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 VL processing helpers."""
from PIL import Image
from transformers import DeepseekV4Config
from vllm.models.deepseek_v4.common.mm_preprocess import (
DeepseekV4VLProcessingInfo,
DeepseekV4VLProcessor,
build_image_block_pad_free,
)
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
VISION_CONFIG = dict(
vision_n_layers=24,
vision_dim=1024,
vision_n_heads=16,
vision_inter_dim=2816,
vision_patch_size=14,
vision_rope_theta=10000.0,
vision_downsample_ratio=3,
vision_max_n_token=384,
vision_min_pixels=147456,
vision_max_wh_ratio=8,
)
class _ConfigCtx:
def __init__(self, config: DeepseekV4Config) -> None:
self._config = config
def get_hf_config(self, *args, **kwargs):
return self._config
def _feature_counts(
processor: DeepseekV4VLProcessor, width: int, height: int
) -> tuple[int, int]:
out = processor(images=[Image.new("RGB", (width, height))])
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
return len(types), out["patches"].shape[0]
def test_get_image_size_with_most_features_beats_wide_real_image():
"""Dummy size must maximize tokens/patches within the budget (#59271).
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
"""
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
processor = DeepseekV4VLProcessor(config)
dummy = info.get_image_size_with_most_features()
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
wide_features = _feature_counts(processor, 1932, 336)
assert dummy == (1932, 336)
assert wide_features <= dummy_features
def test_get_image_size_with_most_features_respects_max_wh_ratio():
config = DeepseekV4Config(**VISION_CONFIG)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
dummy = info.get_image_size_with_most_features()
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
@@ -1,189 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.models.gemma3n_audio_utils import (
adjust_audio_features_to_expected_length,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# Gemma3 (image) model
GEMMA3_MODEL_ID = "google/gemma-3-4b-it"
# Gemma3n (multimodal with audio) model
GEMMA3N_MODEL_ID = "google/gemma-3n-E2B-it"
# Expected audio tokens for Gemma3n (audio_soft_tokens_per_image)
GEMMA3N_EXPECTED_AUDIO_TOKENS = 188
class TestGemma3nAudioTensorLogic:
"""CPU-based tests for Gemma3n audio feature tensor manipulation.
These tests validate the padding/truncation logic in
adjust_audio_features_to_expected_length() which fixes the
integer overflow in _process_audio_input when audio_seq_len > 188.
"""
def test_padding_when_audio_short(self):
"""Test that short audio is padded to expected length."""
batch_size, seq_len, embed_dim = 1, 100, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == 0
# First 100 tokens should be original, rest should be padding (zeros)
assert torch.allclose(result[:, :seq_len, :], audio_features)
assert torch.allclose(
result[:, seq_len:, :],
torch.zeros(batch_size, expected_tokens - seq_len, embed_dim),
)
def test_truncation_when_audio_long(self):
"""Test that long audio is truncated to expected length.
This is the key test for the overflow fix. Previously, when
audio_seq_len > expected_tokens, the code would compute a negative
padding value causing: RuntimeError: numel: integer multiplication overflow
"""
batch_size, seq_len, embed_dim = 1, 192, 256 # 192 > 188
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens # 192 - 188 = 4
# Result should be first 188 tokens of original
assert torch.allclose(result, audio_features[:, :expected_tokens, :])
def test_no_change_when_exact_length(self):
"""Test that exact-length audio passes through unchanged."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, expected_tokens, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == audio_features.shape
assert tokens_truncated == 0
assert torch.allclose(result, audio_features)
def test_original_bug_would_fail(self):
"""Verify the original buggy implementation would cause overflow.
The original code always tried to pad, which fails when
audio_seq_len > expected_tokens because expand() gets negative size.
"""
batch_size, seq_len, embed_dim = 1, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
padding_embs = torch.zeros(1, 1, embed_dim)
# Original buggy logic (always pads, never truncates)
extra_padding_tokens = expected_tokens - seq_len # = -4 (negative!)
with pytest.raises(RuntimeError):
# This should fail with negative size error
padding_embs.expand(batch_size, extra_padding_tokens, embed_dim)
@pytest.mark.parametrize(
"seq_len",
[50, 100, 150, 187, 188, 189, 192, 200, 300],
)
def test_various_audio_lengths(self, seq_len: int):
"""Test padding/truncation with various audio lengths."""
batch_size, embed_dim = 1, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
# Should not raise any errors
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
# Output should always be expected_tokens length
assert result.shape == (batch_size, expected_tokens, embed_dim)
# Verify truncation count is correct
if seq_len > expected_tokens:
assert tokens_truncated == seq_len - expected_tokens
else:
assert tokens_truncated == 0
def test_batch_processing(self):
"""Test that batch processing works correctly."""
batch_size, seq_len, embed_dim = 4, 192, 256
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
audio_features = torch.randn(batch_size, seq_len, embed_dim)
padding_embs = torch.zeros(1, 1, embed_dim)
result, tokens_truncated = adjust_audio_features_to_expected_length(
audio_features, expected_tokens, padding_embs
)
assert result.shape == (batch_size, expected_tokens, embed_dim)
assert tokens_truncated == seq_len - expected_tokens
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
def test_get_image_size_with_most_features(
image_assets: ImageTestAssets,
model_id: str,
mm_processor_kwargs: dict[str, object],
):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
max_image_size = processor.info.get_image_size_with_most_features()
max_tokens = processor.info.get_num_image_tokens(
image_width=max_image_size.width,
image_height=max_image_size.height,
processor=hf_processor,
mm_kwargs=mm_processor_kwargs,
)
prompt = "<start_of_image>"
image_seq_length = hf_processor.image_seq_length
for asset in image_assets:
mm_data = {"image": [asset.pil_image]}
processed_inputs = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs=mm_processor_kwargs,
)
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
num_patches_tensor = mm_kwargs_data["num_patches"]
tokens = int(num_patches_tensor.item()) * image_seq_length
assert tokens <= max_tokens
@@ -1,405 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
from types import SimpleNamespace
import numpy as np
import pytest
import torch
from PIL import Image as PILImage
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import (
Gemma4ForConditionalGeneration,
Gemma4ImagePixelInputs,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFieldConfig
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
def test_gemma4_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
# Production repro: a 3x900 image (extreme aspect ratio) made the
# prompt-side estimator return 289 while the HF Gemma 4 image
# processor's vision tower output capped at 280, producing the
# "Attempted to assign 280 multimodal tokens to 289 placeholders"
# mismatch that crashed EngineCore.
(900, 3, 280),
(3, 900, 280),
# Same pathology should hold for the video-frame budget (70 tokens).
(900, 3, 70),
# And for any other supported budget.
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Regression for the Gemma 3/4 multimodal crash.
`_compute_num_soft_tokens` must never return a value larger than
`max_soft_tokens`. The HF Gemma 4 image processor clamps its vision
tower output to that value; if the prompt-side estimator returns more,
the prompt has more `image` placeholder tokens than the encoder will
fill, and `_merge_multimodal_embeddings` raises `ValueError` deep in
the model forward.
"""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap that the HF "
f"image processor enforces on its vision tower output. This is "
f"the placeholder/encoder count mismatch that crashes EngineCore."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
@pytest.mark.parametrize(
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
)
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
def test_video_cache_is_independent_of_image_kwargs(
model_id: str,
image_kwargs: dict[str, object],
video_uuid: str | None,
):
"""Request image overrides must not change video frames or make results
depend on cache warmth.
"""
request_kwargs = {"images_kwargs": image_kwargs}
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 1, "video": 1},
mm_processor_cache_gb=1,
)
cache = MultiModalProcessorOnlyCache(ctx.model_config)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor = processor.info.get_hf_processor()
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
frames = np.stack([np.asarray(image)] * 2)
metadata = {"fps": 2.0, "frames_indices": [0, 1]}
mm_items = processor.info.parse_mm_data(
{"image": image, "video": [(frames, metadata)]}
)
def process(mm_kwargs, cache):
return processor(
hf_processor.image_token + hf_processor.video_token,
mm_items,
mm_uuid_items={"video": [video_uuid]},
hf_processor_mm_kwargs=mm_kwargs,
cache=cache,
)
# Populate the video cache without image overrides, then change only
# images_kwargs on the next request. Since images_kwargs are excluded from
# the video hash, and must not affect the processed video, the cached result
# must match a fresh recomputation.
baseline = process({}, cache=cache)
cached = process(request_kwargs, cache=cache)
cache.clear_cache()
fresh = process(request_kwargs, cache=cache)
def pixels(result, modality, field):
return result["mm_kwargs"][modality][0][field].data
video_pixels = pixels(fresh, "video", "pixel_values_videos")
assert torch.equal(pixels(cached, "video", "pixel_values_videos"), video_pixels)
assert torch.equal(pixels(baseline, "video", "pixel_values_videos"), video_pixels)
assert baseline["mm_hashes"]["video"] == fresh["mm_hashes"]["video"]
assert not torch.equal(
pixels(baseline, "image", "pixel_values"),
pixels(fresh, "image", "pixel_values"),
)
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
# We only allow 1 image
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
# Provide 2 images in the prompt
prompt = "<image><image>"
# image_assets usually has multiple images
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
# Expect ValueError when exceeding limit
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
# allocating 2.43 GiB int64 inside
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
# memory and counts the ``F.one_hot`` transient as the dominant cost.
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
# Gemma4 vision config default (HF: configuration_gemma4.py).
_POSITION_EMBEDDING_SIZE = 10240
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
_VIDEO_PATCHES_PER_FRAME = 280
def test_encoder_chunk_tight_budget_fits_in_free():
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
total = 22 * GiB_bytes
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
)
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
assert one_hot_bytes <= free // 2
def test_encoder_chunk_roomy_gpu_keeps_batching():
chunk = _encoder_chunk(
_VIDEO_PATCHES_PER_FRAME,
60 * GiB_bytes,
80 * GiB_bytes,
_POSITION_EMBEDDING_SIZE,
)
assert chunk > 8
def test_encoder_chunk_zero_patches_is_safe():
assert (
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
)
def test_encoder_chunk_zero_position_embedding_size_is_safe():
# Degenerate config: must not raise ZeroDivisionError.
assert (
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
)
def test_encoder_chunk_no_free_memory_falls_back_to_one():
assert (
_encoder_chunk(
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
)
== 1
)
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
model = object.__new__(Gemma4ForConditionalGeneration)
vision_config = SimpleNamespace(
patch_size=16, pooling_kernel_size=3, default_output_length=280
)
model.vision_tower = SimpleNamespace(config=vision_config)
model.config = SimpleNamespace(vision_config=vision_config)
return model
@pytest.mark.parametrize(
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
)
def test_encoder_cudagraph_capture_rows_fit_token_budget(
token_budget: int, row_tokens: int
):
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
model = _gemma4_for_encoder_cudagraph()
inputs = model.prepare_encoder_cudagraph_capture_inputs(
token_budget=token_budget,
max_batch_size=58,
max_frames_per_batch=58 * 32,
device="meta",
dtype=torch.bfloat16,
axis_keys=(row_tokens,),
)
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
assert row_patches == row_tokens * 9
assert num_rows == max(1, token_budget // row_tokens)
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
model = _gemma4_for_encoder_cudagraph()
mm_kwargs = {
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
"pixel_position_ids": [
torch.zeros(2520, 2, dtype=torch.long),
torch.zeros(630, 2, dtype=torch.long),
],
}
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
assert [spec.output_tokens for spec in specs] == [280, 70]
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
@@ -1,248 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping
import pytest
import torch
from PIL import Image as PILImage
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
from vllm.exceptions import VLLMValidationError
from vllm.model_executor.models.gemma4_mm import (
Gemma4ImagePixelInputs,
_dummy_audio_num_samples,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig
from ....conftest import ImageTestAssets
from ...utils import build_model_context
# The Unified model ID for testing purposes
GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
Gemma4ImagePixelInputs(
pixel_values=[
torch.randn(10080, 768),
torch.randn(2520, 768),
],
pixel_position_ids=[
torch.zeros(10080, 2, dtype=torch.long),
torch.zeros(2520, 2, dtype=torch.long),
],
)
def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
field = MultiModalFieldConfig.batched("image").field
elems = field.build_elems(
"image",
"pixel_values",
[torch.randn(10080, 768), torch.randn(2520, 768)],
)
reduced = field.reduce_data(list(elems))
assert isinstance(reduced, list)
assert [tensor.shape for tensor in reduced] == [
torch.Size([10080, 768]),
torch.Size([2520, 768]),
]
@pytest.mark.parametrize(
"image_width,image_height,max_soft_tokens",
[
(900, 3, 280),
(3, 900, 280),
(900, 3, 70),
(4000, 2, 1120),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
model_id: str,
image_width: int,
image_height: int,
max_soft_tokens: int,
):
"""Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={"do_pan_and_scan": True},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
num_soft_tokens = processor.info._compute_num_soft_tokens(
image_width=image_width,
image_height=image_height,
max_soft_tokens=max_soft_tokens,
)
assert num_soft_tokens <= max_soft_tokens, (
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
f"image_width={image_width}, image_height={image_height}, "
f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
)
@pytest.mark.parametrize(
("mm_processor_kwargs", "expected_image_tokens"),
[
({}, 280),
({"max_soft_tokens": 70}, 70),
({"max_soft_tokens": 280}, 280),
({"max_soft_tokens": 1120}, 1120),
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
({"images_kwargs": None}, 280),
({"images_kwargs": "not-a-dict"}, 280),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
model_id: str,
mm_processor_kwargs: dict[str, object],
expected_image_tokens: int,
):
ctx = build_model_context(
model_id,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt={"image": 1, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"image": 1, "video": 1},
)
assert tokens is not None
assert tokens["image"] == expected_image_tokens
assert tokens["video"] == 32 * (70 + 2 + 6)
@pytest.mark.parametrize(
("limit_mm_per_prompt", "expected_video_tokens"),
[
({"video": 1}, 32 * (70 + 2 + 6)),
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
],
)
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
model_id: str,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
expected_video_tokens: int,
):
ctx = build_model_context(
model_id,
limit_mm_per_prompt=limit_mm_per_prompt,
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
tokens = processor.info.get_mm_max_tokens_per_item(
seq_len=ctx.model_config.max_model_len,
mm_counts={"video": 1},
)
assert tokens is not None
assert tokens["image"] == 280
assert tokens["video"] == expected_video_tokens
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
ctx = build_model_context(
model_id,
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = PILImage.new("RGB", (1000, 1000), color="white")
image_size = image.size
mm_items = processor.info.parse_mm_data({"image": image})
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
replacement = prompt_update.resolve(0).content.full
expected = processor.info.get_image_repl(
image_width=image_size[0],
image_height=image_size[1],
processor=processor.info.get_hf_processor(),
max_soft_tokens=560,
).full
assert replacement == expected
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
def test_limit_mm_per_prompt(
image_assets: ImageTestAssets,
model_id: str,
):
"""Test that limit_mm_per_prompt restricts multiple images correctly."""
ctx = build_model_context(
model_id,
mm_processor_kwargs={},
limit_mm_per_prompt={"image": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<image><image>"
images = [asset.pil_image for asset in image_assets][:2]
if len(images) < 2:
images = [images[0], images[0]]
mm_data = {"image": images}
with pytest.raises(VLLMValidationError, match="At most 1 image"):
processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
"""The unified audio feature extractor does not define ``fft_length``.
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
unconditionally, so memory profiling raised ``AttributeError`` for every
unified Gemma 4 deployment in which audio is the modality with the largest
per-item token count -- which is the case as soon as ``audio_seq_length``
is raised above the video budget.
"""
class _UnifiedFeatureExtractor:
sampling_rate = 16000
class _UnifiedProcessor:
feature_extractor = _UnifiedFeatureExtractor()
audio_seq_length = 3000
audio_ms_per_token = 40
# 3000 tokens * 40 ms = 120 s at 16 kHz
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
"""The mel/tower extractor keeps its existing behaviour."""
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
self.fft_length = 512
self.sampling_rate = 16000
class _TowerProcessor:
feature_extractor = _TowerFeatureExtractor()
audio_seq_length = 750
audio_ms_per_token = 40
assert _dummy_audio_num_samples(_TowerProcessor()) == 512

Some files were not shown because too many files have changed in this diff Show More