[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,148 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.deepseek_v32.nvidia import model as deepseek_v32_model
|
||||
from vllm.models.deepseek_v32.nvidia import mtp as deepseek_v32_mtp
|
||||
|
||||
|
||||
class _IdentityNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
|
||||
self.variance_epsilon = 1e-5
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _RecordingModule(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor, **kwargs) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _RecordingProjection(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(2, 4), requires_grad=False)
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states[:, :2]
|
||||
|
||||
|
||||
class _SequenceParallelMTPBlock:
|
||||
use_sequence_parallel = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
):
|
||||
assert residual is None
|
||||
return hidden_states * 2, hidden_states * 3
|
||||
|
||||
|
||||
def _mock_sequence_parallel_collectives(monkeypatch, module):
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_reduce_scatter",
|
||||
lambda tensor: tensor.chunk(2, dim=0)[0],
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
|
||||
def test_decoder_layer_keeps_dense_states_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(deepseek_v32_model.DeepseekV32DecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_sequence_parallel = True
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.self_attn = _RecordingModule()
|
||||
layer.mlp = _RecordingModule()
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_model)
|
||||
|
||||
positions = torch.arange(3)
|
||||
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states = deepseek_v32_model.sp_shard(full_hidden_states)
|
||||
hidden_states, residual = layer(positions, hidden_states, residual=None)
|
||||
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.self_attn.num_tokens == 3
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.self_attn.num_tokens == 3
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_mtp_projects_sequence_shard_and_restores_full_output(monkeypatch):
|
||||
layer = object.__new__(deepseek_v32_mtp.DeepseekV32MultiTokenPredictorLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.enorm = _IdentityNorm()
|
||||
layer.hnorm = _IdentityNorm()
|
||||
layer.eh_proj = _RecordingProjection()
|
||||
layer._eh_plan = None
|
||||
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
|
||||
norm = Mock(
|
||||
side_effect=lambda hidden_states, residual: (hidden_states + residual, None)
|
||||
)
|
||||
object.__setattr__(layer, "shared_head", SimpleNamespace(norm=norm))
|
||||
|
||||
monkeypatch.setattr(
|
||||
deepseek_v32_mtp,
|
||||
"fused_eh_norm",
|
||||
lambda positions, inputs_embeds, previous_hidden_states, *args: torch.cat(
|
||||
[inputs_embeds, previous_hidden_states], dim=-1
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(deepseek_v32_mtp, "run_glm52_plan", lambda *args: None)
|
||||
_mock_sequence_parallel_collectives(monkeypatch, deepseek_v32_mtp)
|
||||
|
||||
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states, recycled_hidden_states = layer(
|
||||
input_ids=torch.zeros(3, dtype=torch.long),
|
||||
positions=torch.arange(3),
|
||||
previous_hidden_states=torch.zeros_like(inputs_embeds),
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
|
||||
expected = torch.cat([sharded_states * 5, sharded_states * 5])[:3]
|
||||
assert layer.eh_proj.num_tokens == 2
|
||||
torch.testing.assert_close(hidden_states, expected)
|
||||
torch.testing.assert_close(recycled_hidden_states, expected)
|
||||
norm.assert_called_once()
|
||||
@@ -1,432 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash KDA recurrent (decode) kernel.
|
||||
|
||||
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
|
||||
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
|
||||
token-strided rather than contiguous. Both backends must match a pure-PyTorch
|
||||
recurrence, read these slices in place and reject unsupported layouts.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.third_party.kda import fused_recurrent_kda
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia.ops.third_party.kda import fused_recurrent_kda
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
|
||||
)
|
||||
|
||||
H, D = 16, 128
|
||||
LOWER_BOUND = -5.0
|
||||
|
||||
|
||||
def naive_recurrent_kda(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
a_log: torch.Tensor,
|
||||
g_bias: torch.Tensor,
|
||||
state: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""fp32 reference for one sequence: ``[T, H, D]`` inputs, ``[H, D, D]``
|
||||
(v-major) state; mirrors the kernel's in-kernel gate, beta sigmoid and
|
||||
q/k l2norm.
|
||||
"""
|
||||
q, k, v, raw_g, raw_beta = (x.float() for x in (q, k, v, raw_g, raw_beta))
|
||||
q = q / torch.sqrt(q.square().sum(-1, keepdim=True) + 1e-6) * D**-0.5
|
||||
k = k / torch.sqrt(k.square().sum(-1, keepdim=True) + 1e-6)
|
||||
gate = LOWER_BOUND * torch.sigmoid(a_log.exp()[:, None] * (raw_g + g_bias))
|
||||
beta = torch.sigmoid(raw_beta)
|
||||
s = state.clone()
|
||||
out = torch.empty_like(v)
|
||||
for t in range(q.shape[0]):
|
||||
s = s * gate[t].exp()[:, None, :]
|
||||
u = beta[t][:, None] * (v[t] - torch.einsum("hvk,hk->hv", s, k[t]))
|
||||
s = s + u[:, :, None] * k[t][:, None, :]
|
||||
out[t] = torch.einsum("hvk,hk->hv", s, q[t])
|
||||
return out, s
|
||||
|
||||
|
||||
def make_inputs(num_seqs: int, query_len: int, device: torch.device):
|
||||
"""Token-strided q/k/v/beta as the decode path produces them: column
|
||||
slices of a merged ``[T, q|k|v]`` conv output and of the fused
|
||||
``[T, qkv|beta|f_a|g_a]`` projection.
|
||||
"""
|
||||
T, proj = num_seqs * query_len, H * D
|
||||
qkv = torch.randn(T, 3 * proj, dtype=torch.bfloat16, device=device)
|
||||
projected = torch.randn(
|
||||
T, 3 * proj + H + 2 * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
q, k, v = (x.view(1, T, H, D) for x in qkv.split(proj, dim=-1))
|
||||
beta = projected[:, 3 * proj : 3 * proj + H].unsqueeze(0)
|
||||
# (A size-1 token dim gets an arbitrary stride from `view`.)
|
||||
assert T == 1 or (q.stride(1) == 3 * proj and beta.stride(1) == projected.stride(0))
|
||||
inputs = dict(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=torch.randn(1, T, H, D, dtype=torch.bfloat16, device=device),
|
||||
beta=beta,
|
||||
a_log=0.5 * torch.randn(H, dtype=torch.float32, device=device),
|
||||
g_bias=0.1 * torch.randn(H * D, dtype=torch.float32, device=device),
|
||||
cu_seqlens=torch.arange(0, T + 1, query_len, dtype=torch.int32, device=device),
|
||||
)
|
||||
# Slot 0 is NULL_BLOCK_ID; sequences own random distinct slots (one per
|
||||
# token in the spec-decode layout).
|
||||
slots = torch.randperm(T, device=device).to(torch.int32) + 1
|
||||
if query_len == 1:
|
||||
inputs["ssm_state_indices"] = slots
|
||||
else:
|
||||
inputs["ssm_state_indices"] = slots.view(num_seqs, query_len)
|
||||
inputs["num_accepted_tokens"] = torch.randint(
|
||||
1, query_len + 1, (num_seqs,), dtype=torch.int32, device=device
|
||||
)
|
||||
state = torch.randn(T + 1, H, D, D, dtype=torch.float32, device=device)
|
||||
return inputs, state
|
||||
|
||||
|
||||
def run_kernel(inputs: dict, state: torch.Tensor) -> torch.Tensor:
|
||||
out, _ = fused_recurrent_kda(
|
||||
**inputs,
|
||||
initial_state=state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
sigmoid_beta=True,
|
||||
compute_gate=True,
|
||||
lower_bound=LOWER_BOUND,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_seqs", "query_len"), [(1, 1), (7, 1), (3, 3)], ids=["1x1", "7x1", "3x3"]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_recurrent_kda_matches_reference(num_seqs: int, query_len: int):
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
inputs, state = make_inputs(num_seqs, query_len, device)
|
||||
expected_state = state.clone()
|
||||
out = run_kernel(inputs, state)
|
||||
|
||||
indices = inputs["ssm_state_indices"].view(num_seqs, query_len)
|
||||
accepted = inputs.get("num_accepted_tokens")
|
||||
expected = torch.empty_like(out[0], dtype=torch.float32)
|
||||
for n in range(num_seqs):
|
||||
first = indices[n, 0 if accepted is None else accepted[n] - 1]
|
||||
s = expected_state[first]
|
||||
for t in range(query_len):
|
||||
tok = slice(n * query_len + t, n * query_len + t + 1)
|
||||
expected[tok], s = naive_recurrent_kda(
|
||||
inputs["q"][0, tok],
|
||||
inputs["k"][0, tok],
|
||||
inputs["v"][0, tok],
|
||||
inputs["g"][0, tok],
|
||||
inputs["beta"][0, tok],
|
||||
inputs["a_log"],
|
||||
inputs["g_bias"].view(H, D),
|
||||
s,
|
||||
)
|
||||
expected_state[indices[n, t]] = s
|
||||
|
||||
torch.testing.assert_close(out[0].float(), expected, rtol=1e-2, atol=1e-3)
|
||||
used = indices.flatten().long()
|
||||
torch.testing.assert_close(state[used], expected_state[used], rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_seqs", "query_len"), [(7, 1), (3, 3)], ids=["7x1", "3x3"]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
|
||||
num_seqs: int, query_len: int
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
inputs, state = make_inputs(num_seqs, query_len, device)
|
||||
for name in ("q", "k", "v", "beta"):
|
||||
assert not inputs[name].is_contiguous()
|
||||
contiguous = {
|
||||
name: x.contiguous() if name in ("q", "k", "v", "beta") else x
|
||||
for name, x in inputs.items()
|
||||
}
|
||||
state_ref = state.clone()
|
||||
out_ref = run_kernel(contiguous, state_ref)
|
||||
out = run_kernel(inputs, state)
|
||||
torch.testing.assert_close(out, out_ref, rtol=0, atol=0)
|
||||
torch.testing.assert_close(state, state_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Both implementations require zero-copy token-strided layouts",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
|
||||
"""Layouts the token-stride addressing cannot express must fail loudly
|
||||
rather than read the wrong tokens: a batch slice of a wider buffer
|
||||
(``stride(0) != T * stride(1)``), overlapping tokens, and a head-strided
|
||||
(transposed) block.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
inputs, state = make_inputs(1, 1, device)
|
||||
T = 4
|
||||
base = torch.randn(4, T, 2 * H * D, dtype=torch.bfloat16, device=device)
|
||||
bad_q = {
|
||||
"batch slice": base[::2, :, : H * D].view(2, T, H, D),
|
||||
"overlapping tokens": base[:1, :, : H * D].as_strided(
|
||||
(1, T, H, D), (0, D, D, 1)
|
||||
),
|
||||
"head-strided": base[:1, :, : H * D].view(1, T, D, H).transpose(2, 3),
|
||||
}
|
||||
for q in bad_q.values():
|
||||
broken = dict(inputs, q=q, k=q, v=q)
|
||||
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
|
||||
with pytest.raises(AssertionError, match=r"torch.Size"):
|
||||
run_kernel(broken, state)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="The large-grid fix is NVIDIA-only"
|
||||
)
|
||||
@pytest.mark.parametrize("num_seqs,num_heads", [(1024, 64), (2048, 48)])
|
||||
def test_fused_recurrent_kda_more_than_65535_programs(num_seqs: int, num_heads: int):
|
||||
"""Regression test for the launch grid.
|
||||
|
||||
GLM-5.3-Flash has 64 KDA value heads; at TP=1 (every DP rank) a decode
|
||||
batch of 1024 sequences needs 1024 * 64 = 65536 (sequence, head)
|
||||
programs. The kernel used to put that product into gridDim.z, whose CUDA
|
||||
limit is 65535, so the launch failed with ``Triton Error [CUDA]: invalid
|
||||
argument`` at CUDA-graph capture and DP deployments could not start. The
|
||||
grid now carries the product in gridDim.x. Small head dims keep the test
|
||||
cheap; the shape that matters is the number of programs. Every (sequence,
|
||||
head) pair is independent, so the full batch must be bitwise identical to
|
||||
the same batch run in two halves.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
dev = torch.device("cuda")
|
||||
head_dim = 16
|
||||
pool = num_seqs + 8
|
||||
q = torch.randn(1, num_seqs, num_heads, head_dim, device=dev, dtype=torch.bfloat16)
|
||||
k = torch.randn_like(q)
|
||||
v = torch.randn_like(q)
|
||||
g = torch.randn_like(q)
|
||||
beta = torch.randn(1, num_seqs, num_heads, device=dev, dtype=torch.bfloat16)
|
||||
state = torch.randn(
|
||||
pool, num_heads, head_dim, head_dim, device=dev, dtype=torch.float32
|
||||
)
|
||||
cu = torch.arange(0, num_seqs + 1, device=dev, dtype=torch.int32)
|
||||
# slot 0 is NULL_BLOCK_ID and is skipped by the kernel: use slots 1..
|
||||
idx = (torch.randperm(pool - 1, device=dev)[:num_seqs] + 1).to(torch.int32)
|
||||
a_log = torch.randn(num_heads, device=dev, dtype=torch.float32)
|
||||
g_bias = torch.randn(num_heads, head_dim, device=dev, dtype=torch.float32)
|
||||
|
||||
def run(sl: slice, cu_: torch.Tensor, st: torch.Tensor, out: torch.Tensor):
|
||||
fused_recurrent_kda(
|
||||
q=q[:, sl],
|
||||
k=k[:, sl],
|
||||
v=v[:, sl],
|
||||
g=g[:, sl],
|
||||
beta=beta[:, sl],
|
||||
initial_state=st,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=cu_,
|
||||
ssm_state_indices=idx[sl],
|
||||
sigmoid_beta=True,
|
||||
a_log=a_log,
|
||||
g_bias=g_bias,
|
||||
compute_gate=True,
|
||||
lower_bound=-5.0,
|
||||
out=out,
|
||||
)
|
||||
|
||||
full_state = state.clone()
|
||||
full_out = torch.zeros_like(k)
|
||||
run(slice(0, num_seqs), cu, full_state, full_out)
|
||||
|
||||
half = num_seqs // 2
|
||||
ref_state = state.clone()
|
||||
ref_out = torch.zeros_like(k)
|
||||
for s, e in ((0, half), (half, num_seqs)):
|
||||
cu_h = torch.arange(0, e - s + 1, device=dev, dtype=torch.int32)
|
||||
part = torch.zeros_like(k[:, s:e])
|
||||
run(slice(s, e), cu_h, ref_state, part)
|
||||
ref_out[:, s:e] = part
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(full_out, ref_out)
|
||||
assert torch.equal(full_state, ref_state)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [16, 144, 256])
|
||||
@pytest.mark.parametrize("dim_first", [False, True])
|
||||
@pytest.mark.parametrize("num_spec", [0, 3])
|
||||
@torch.inference_mode()
|
||||
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
|
||||
"""Restoring both cached states must reproduce an uninterrupted prefill."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.glm5next.common import kda
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
|
||||
pytest.skip("requires FlashKDA")
|
||||
import vllm._flashkda_C # noqa: F401
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
length, heads = 273, 2
|
||||
width, state_len = 3 * heads * D, 3 + num_spec
|
||||
layer = object.__new__(kda.Glm5NextLinearAttention)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.prefix = "layer.0"
|
||||
layer.local_num_heads = heads
|
||||
layer.head_dim = D
|
||||
layer.conv_size = 4
|
||||
layer.local_projection_size = heads * D
|
||||
layer.kda_safe_gate = True
|
||||
layer.kda_lower_bound = -5.0
|
||||
layer.kda_prefill_backend = "flashkda"
|
||||
layer._conv_state_dim_first = dim_first
|
||||
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
|
||||
layer.q_conv1d = SimpleNamespace(bias=None)
|
||||
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
|
||||
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
|
||||
state_shape = (2, heads, D, D)
|
||||
layer._flashkda_buffer_specs = (
|
||||
(state_shape, torch.float32),
|
||||
(state_shape, torch.float32),
|
||||
(
|
||||
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
|
||||
torch.uint8,
|
||||
),
|
||||
((1, 2 * length, heads, D), torch.bfloat16),
|
||||
)
|
||||
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
|
||||
workspace = WorkspaceManager(device)
|
||||
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
|
||||
conv_shape = (width, state_len) if dim_first else (state_len, width)
|
||||
num_slots = 5 + (num_spec + 1 if num_spec else 0)
|
||||
conv_storage = torch.zeros(
|
||||
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
|
||||
recurrent_storage = torch.zeros(
|
||||
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
|
||||
)
|
||||
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
|
||||
layer.kv_cache = (conv, recurrent)
|
||||
projected = torch.randn(
|
||||
2 * length, width + heads, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
|
||||
raw_before = raw_qkv.clone()
|
||||
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
|
||||
|
||||
def run(qkv, g, b, slots, has_initial, checkpoint=None):
|
||||
n = qkv.shape[0] // 2
|
||||
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
|
||||
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
|
||||
cu_cpu, device=device
|
||||
)
|
||||
metadata = GDNAttentionMetadata(
|
||||
num_prefills=2,
|
||||
num_prefill_tokens=2 * n,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_spec_decodes=0,
|
||||
num_spec_decode_tokens=0,
|
||||
num_actual_tokens=2 * n,
|
||||
has_initial_state=torch.tensor(has_initial, device=device),
|
||||
non_spec_query_start_loc=cu_cpu.to(device),
|
||||
non_spec_state_indices_tensor=torch.tensor(
|
||||
slots, dtype=torch.int32, device=device
|
||||
),
|
||||
checkpoint=checkpoint,
|
||||
nums_dict=nums,
|
||||
batch_ptr=batch_ptr,
|
||||
token_chunk_offset_ptr=chunk_offsets,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kda,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
|
||||
)
|
||||
spec_tokens = num_spec + 1 if num_spec else 0
|
||||
if spec_tokens:
|
||||
qkv = torch.cat([qkv[:spec_tokens], qkv])
|
||||
g = torch.cat([g[:, :spec_tokens], g], dim=1)
|
||||
b = torch.cat([b[:, :spec_tokens], b], dim=1)
|
||||
metadata.num_spec_decodes = 1
|
||||
metadata.num_spec_decode_tokens = spec_tokens
|
||||
metadata.num_actual_tokens += spec_tokens
|
||||
metadata.spec_sequence_masks = torch.tensor(
|
||||
[True, False, False], device=device
|
||||
)
|
||||
metadata.spec_query_start_loc = torch.tensor(
|
||||
[0, spec_tokens], dtype=torch.int32, device=device
|
||||
)
|
||||
metadata.spec_state_indices_tensor = torch.arange(
|
||||
5, num_slots, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
|
||||
metadata.non_spec_token_indx = torch.arange(
|
||||
spec_tokens, qkv.shape[0], device=device
|
||||
)
|
||||
metadata.num_accepted_tokens = torch.ones(
|
||||
1, dtype=torch.int32, device=device
|
||||
)
|
||||
out = torch.empty(
|
||||
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
layer._forward(qkv, g, b, out)
|
||||
return out[:, spec_tokens:]
|
||||
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
torch.tensor([offset, 0], dtype=torch.int32, device=device),
|
||||
torch.tensor([3, 0], dtype=torch.int32, device=device),
|
||||
)
|
||||
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
|
||||
expected_state = recurrent[1].clone()
|
||||
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
|
||||
torch.testing.assert_close(raw_qkv, raw_before)
|
||||
conv_view = conv if dim_first else conv.transpose(-1, -2)
|
||||
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
|
||||
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
|
||||
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
|
||||
|
||||
run(
|
||||
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
|
||||
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
|
||||
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
|
||||
[1, 2],
|
||||
[False, False],
|
||||
)
|
||||
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
|
||||
|
||||
suffix = slice(offset, length)
|
||||
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
|
||||
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
|
||||
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
|
||||
conv[4].copy_(saved_conv)
|
||||
recurrent[4].copy_(saved_recurrent)
|
||||
actual = run(qkv, g, b, [3, 4], [True, True])
|
||||
torch.testing.assert_close(
|
||||
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
|
||||
)
|
||||
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""``--mamba-ssm-cache-dtype`` must reach GLM-5.3-Flash's KDA recurrent state
|
||||
(it used to be dropped, leaving the state float32 unconditionally)."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.glm5next.common.kda import Glm5NextLinearAttention
|
||||
from vllm.models.glm5next.common.model import (
|
||||
Glm5NextForCausalLM,
|
||||
Glm5NextForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ssm_dtype", "expected"),
|
||||
[("auto", torch.float32), ("float32", torch.float32), ("bfloat16", torch.bfloat16)],
|
||||
)
|
||||
def test_kda_recurrent_state_dtype_follows_ssm_cache_dtype(ssm_dtype, expected):
|
||||
model_config = SimpleNamespace(dtype=torch.bfloat16)
|
||||
cache_config = SimpleNamespace(
|
||||
mamba_cache_dtype="auto",
|
||||
mamba_ssm_cache_dtype=ssm_dtype,
|
||||
use_recoverssm=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(model_config=model_config, cache_config=cache_config)
|
||||
want = (torch.bfloat16, expected)
|
||||
|
||||
assert Glm5NextForCausalLM.get_mamba_state_dtype_from_config(vllm_config) == want
|
||||
assert (
|
||||
Glm5NextForConditionalGeneration.get_mamba_state_dtype_from_config(vllm_config)
|
||||
== want
|
||||
)
|
||||
# The layer's own dtype must agree with the model-level one, since the
|
||||
# KV-cache spec and the kernels read them separately.
|
||||
layer = Glm5NextLinearAttention.__new__(Glm5NextLinearAttention)
|
||||
layer.model_config, layer.cache_config = model_config, cache_config
|
||||
assert layer.get_state_dtype() == want
|
||||
@@ -1,114 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash sequence-parallel MoE layout.
|
||||
|
||||
With DP > 1, TP > 1 and expert parallelism, ``Glm5NextModel`` shards the token
|
||||
dimension across the TP group once at the model entry and every layer runs its
|
||||
MLP on that shard. A module that still does tensor-parallel collectives there
|
||||
sums the partial results of *different* tokens held by different ranks, so the
|
||||
dense MLP of the first ``first_k_dense_replace`` layers must hold replicated
|
||||
weights and run without any collective, like the shared experts already do.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Glm5NextTextConfig
|
||||
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.layers import linear
|
||||
from vllm.models.glm5next.common import model as glm_model
|
||||
|
||||
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
|
||||
|
||||
|
||||
class _Attention(nn.Module):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__()
|
||||
self.o_proj = SimpleNamespace(reduce_results=True)
|
||||
|
||||
|
||||
def _fake_tensor_parallel_world(monkeypatch) -> None:
|
||||
"""Pretend to be rank 1 of a TP=2 group whose all-reduce must never run."""
|
||||
|
||||
def _forbidden_all_reduce(*args, **kwargs):
|
||||
raise AssertionError("tensor-parallel all-reduce on a sequence-parallel shard")
|
||||
|
||||
for module in (linear, parameter):
|
||||
monkeypatch.setattr(
|
||||
module, "get_tensor_model_parallel_world_size", lambda: TP_SIZE
|
||||
)
|
||||
monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
linear, "tensor_model_parallel_all_reduce", _forbidden_all_reduce
|
||||
)
|
||||
|
||||
|
||||
def _reference_mlp(mlp: glm_model.Glm5NextMLP, x: torch.Tensor) -> torch.Tensor:
|
||||
gate_up = x @ mlp.gate_up_proj.weight.T
|
||||
gate, up = gate_up.split(INTERMEDIATE, dim=-1)
|
||||
return (torch.nn.functional.silu(gate) * up) @ mlp.down_proj.weight.T
|
||||
|
||||
|
||||
def _assert_runs_locally_on_shard(mlp: glm_model.Glm5NextMLP) -> None:
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
mlp.to(device)
|
||||
assert mlp.gate_up_proj.weight.shape == (2 * INTERMEDIATE, HIDDEN)
|
||||
assert mlp.down_proj.weight.shape == (HIDDEN, INTERMEDIATE)
|
||||
torch.manual_seed(0)
|
||||
for param in mlp.parameters():
|
||||
param.data.normal_()
|
||||
shard = torch.randn(3, HIDDEN, device=device)
|
||||
torch.testing.assert_close(mlp(shard), _reference_mlp(mlp, shard))
|
||||
|
||||
|
||||
def test_dense_mlp_runs_locally_on_the_sequence_parallel_shard(
|
||||
monkeypatch, default_vllm_config
|
||||
):
|
||||
_fake_tensor_parallel_world(monkeypatch)
|
||||
|
||||
sharded = glm_model.Glm5NextMLP(HIDDEN, INTERMEDIATE, "silu")
|
||||
assert sharded.down_proj.tp_size == TP_SIZE
|
||||
assert sharded.gate_up_proj.weight.shape == (2 * INTERMEDIATE // TP_SIZE, HIDDEN)
|
||||
|
||||
_assert_runs_locally_on_shard(
|
||||
glm_model.Glm5NextMLP(HIDDEN, INTERMEDIATE, "silu", is_sequence_parallel=True)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_sequence_parallel_moe", [True, False])
|
||||
def test_dense_layer_mlp_follows_the_sequence_parallel_layout(
|
||||
monkeypatch, default_vllm_config, use_sequence_parallel_moe: bool
|
||||
):
|
||||
_fake_tensor_parallel_world(monkeypatch)
|
||||
monkeypatch.setattr(glm_model, "Glm5NextLinearAttention", _Attention)
|
||||
monkeypatch.setattr(glm_model, "Glm5NextMLAAttention", _Attention)
|
||||
|
||||
config = Glm5NextTextConfig(
|
||||
hidden_size=HIDDEN,
|
||||
intermediate_size=INTERMEDIATE,
|
||||
num_hidden_layers=2,
|
||||
first_k_dense_replace=1,
|
||||
n_routed_experts=4,
|
||||
layer_types=["linear_attention", "deepseek_sparse_attention"],
|
||||
mhc=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
cache_config=None,
|
||||
quant_config=None,
|
||||
parallel_config=SimpleNamespace(
|
||||
use_sequence_parallel_moe=use_sequence_parallel_moe
|
||||
),
|
||||
kernel_config=SimpleNamespace(enable_jit_warmup=False),
|
||||
)
|
||||
|
||||
layer = glm_model.Glm5NextDecoderLayer(vllm_config, config, layer_idx=0)
|
||||
|
||||
assert isinstance(layer.mlp, glm_model.Glm5NextMLP)
|
||||
assert layer.self_attn.o_proj.reduce_results is not use_sequence_parallel_moe
|
||||
if use_sequence_parallel_moe:
|
||||
_assert_runs_locally_on_shard(layer.mlp)
|
||||
else:
|
||||
assert layer.mlp.down_proj.tp_size == TP_SIZE
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the GLM kpool indexer's top-k backend wiring."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _require_deep_gemm() -> None:
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
|
||||
if not has_deep_gemm():
|
||||
pytest.skip("kpool indexer requires DeepGEMM")
|
||||
|
||||
|
||||
def _build(indexer_cls, backend: str):
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
return indexer_cls(
|
||||
k_cache=None,
|
||||
quant_block_size=128,
|
||||
scale_fmt="ue8m0",
|
||||
topk_tokens=2048,
|
||||
head_dim=128,
|
||||
max_pool_len=4096,
|
||||
max_total_seq_len=8192,
|
||||
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
|
||||
backend: str,
|
||||
) -> None:
|
||||
"""The AMD kpool indexer must go through the same dispatcher, so the
|
||||
AITER decode top-k is reachable from kernel_config."""
|
||||
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize(
|
||||
"configured,full_cudagraph,expected",
|
||||
[
|
||||
("auto", False, "aiter"),
|
||||
("auto", True, "auto"),
|
||||
("aiter", True, "aiter"),
|
||||
("per_row", False, "per_row"),
|
||||
],
|
||||
)
|
||||
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: str,
|
||||
full_cudagraph: bool,
|
||||
expected: str,
|
||||
) -> None:
|
||||
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
|
||||
use the context length to pick AITER and must stay on the in-tree kernel.
|
||||
Explicit backends keep working, and outside FULL the long-context heuristic
|
||||
still narrows to AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.models.glm5next.amd import sparse_indexer
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_indexer_top_k_supported",
|
||||
lambda **kwargs: True,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
assert (
|
||||
sparse_indexer._kpool_decode_topk_backend(
|
||||
configured,
|
||||
num_rows=8,
|
||||
max_valid_seq_len=64 * 1024,
|
||||
select_k=512,
|
||||
index_kpool=4,
|
||||
full_cudagraph=full_cudagraph,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
@@ -1,59 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm guards for shared Inkling model-definition contracts."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.models.inkling as inkling
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.models.interfaces import supports_lora
|
||||
from vllm.models.inkling.amd import model as amd_model
|
||||
from vllm.models.inkling.amd.mtp import InklingMTP
|
||||
|
||||
|
||||
def test_backend_neutral_definitions_match_nvidia_reference() -> None:
|
||||
inkling_dir = Path(amd_model.__file__).parents[1]
|
||||
for filename in ("logits_processor.py", "mtp.py"):
|
||||
assert (inkling_dir / "amd" / filename).read_bytes() == (
|
||||
inkling_dir / "nvidia" / filename
|
||||
).read_bytes()
|
||||
|
||||
|
||||
def test_rocm_exports_its_aligned_mtp_implementation() -> None:
|
||||
assert inkling.InklingMTP is InklingMTP
|
||||
|
||||
|
||||
def test_rocm_model_exposes_the_upstream_lora_contract() -> None:
|
||||
model_cls = amd_model._TmlForCausalLMBase
|
||||
|
||||
assert supports_lora(model_cls)
|
||||
assert model_cls.packed_modules_mapping == {
|
||||
"qkvr": ["wq_du", "wk_dv", "wv_dv", "wr_du"],
|
||||
"w13": ["w1", "w3"],
|
||||
}
|
||||
assert model_cls.embedding_modules == {"lm_head": "output_embeddings"}
|
||||
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = model_cls.embedding_modules
|
||||
supported = get_supported_lora_modules(model)
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
def test_lightseek_bundled_adapter_weights_remain_opt_in() -> None:
|
||||
mapper = amd_model._TmlForCausalLMBase.hf_to_vllm_mapper
|
||||
weight = torch.empty(1)
|
||||
name, mapped_weight = next(
|
||||
iter(
|
||||
mapper.apply([("language_model.layers.3.attn.wq_du.lora_A.weight", weight)])
|
||||
)
|
||||
)
|
||||
|
||||
assert name == "model.layers.3.attn.qkvr.lora_A.weight"
|
||||
assert mapped_weight.shard_id == 0
|
||||
assert amd_model._is_peft_adapter_weight(name)
|
||||
|
||||
lm_head_name = mapper.apply_list(["language_model.lm_head.lora_B.weight"])
|
||||
assert lm_head_name == ["lm_head.lora_B.weight"]
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for Quark OCP MXFP4 weights loaded into AITER MoE layouts."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.moe import InklingMoE
|
||||
|
||||
|
||||
def _fake_moe() -> tuple[InklingMoE, SimpleNamespace]:
|
||||
moe = InklingMoE.__new__(InklingMoE)
|
||||
nn.Module.__init__(moe)
|
||||
moe.n_routed_experts = 4
|
||||
routed = SimpleNamespace(
|
||||
moe_config=SimpleNamespace(
|
||||
moe_parallel_config=SimpleNamespace(tp_rank=1, tp_size=2)
|
||||
),
|
||||
# Logical intermediate per rank is 3; AITER pads it to 4.
|
||||
w13_weight=nn.Parameter(torch.zeros(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight=nn.Parameter(torch.zeros(4, 5, 3, dtype=torch.uint8), False),
|
||||
w13_weight_scale=nn.Parameter(torch.ones(4, 8, 2, dtype=torch.uint8), False),
|
||||
w2_weight_scale=nn.Parameter(torch.ones(4, 5, 3, dtype=torch.uint8), False),
|
||||
)
|
||||
moe.experts = SimpleNamespace(routed_experts=routed)
|
||||
moe._local_expert_slots = ( # type: ignore[method-assign]
|
||||
lambda: dict(enumerate(range(4)))
|
||||
)
|
||||
return moe, routed
|
||||
|
||||
|
||||
def test_loads_logical_tp_shards_and_preserves_aiter_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
w13 = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4, 12, 2)
|
||||
w2 = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4, 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight", w13)
|
||||
moe.load_expert_weight("experts.w2_weight", w2)
|
||||
|
||||
# TP rank 1 consumes the second six interleaved gate/up rows. AITER keeps
|
||||
# one padding row in each destination half and one padding w2 column.
|
||||
torch.testing.assert_close(routed.w13_weight[:, :3], w13[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight[:, 4:7], w13[:, 7:12:2])
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 3]) == 0
|
||||
assert torch.count_nonzero(routed.w13_weight[:, 7]) == 0
|
||||
torch.testing.assert_close(routed.w2_weight[:, :, :2], w2[:, :, 2:4])
|
||||
assert torch.count_nonzero(routed.w2_weight[:, :, 2]) == 0
|
||||
|
||||
|
||||
def test_unflattens_quark_scales_and_preserves_scale_padding():
|
||||
moe, routed = _fake_moe()
|
||||
|
||||
flat_w13_scale = torch.arange(4 * 12 * 2, dtype=torch.uint8).view(4 * 12, 2)
|
||||
flat_w2_scale = torch.arange(4 * 5 * 4, dtype=torch.uint8).view(4 * 5, 4)
|
||||
moe.load_expert_weight("experts.w13_weight_scale", flat_w13_scale)
|
||||
moe.load_expert_weight("experts.w2_weight_scale", flat_w2_scale)
|
||||
|
||||
w13_scale = flat_w13_scale.view(4, 12, 2)
|
||||
w2_scale = flat_w2_scale.view(4, 5, 4)
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, :3], w13_scale[:, 6:12:2])
|
||||
torch.testing.assert_close(routed.w13_weight_scale[:, 4:7], w13_scale[:, 7:12:2])
|
||||
assert torch.all(routed.w13_weight_scale[:, 3] == 1)
|
||||
assert torch.all(routed.w13_weight_scale[:, 7] == 1)
|
||||
torch.testing.assert_close(routed.w2_weight_scale[:, :, :2], w2_scale[:, :, 2:4])
|
||||
assert torch.all(routed.w2_weight_scale[:, :, 2] == 1)
|
||||
@@ -1,425 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for Inkling's ROCm relative-bias paged attention."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.fa4_rel_attention import (
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_rel_attention,
|
||||
use_gfx950_gluon_decode,
|
||||
use_gfx950_gluon_extend,
|
||||
)
|
||||
from vllm.models.inkling.amd.ops.rel_attention_decode import (
|
||||
decode_split_count,
|
||||
use_split_kv_decode,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _reference(
|
||||
q: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
rel_logits: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
q_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
) -> torch.Tensor:
|
||||
num_heads = q.shape[1]
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
gqa_group = num_heads // num_kv_heads
|
||||
out: list[torch.Tensor] = []
|
||||
q_start = 0
|
||||
|
||||
for req, (q_len, kv_len) in enumerate(zip(q_lens, kv_lens)):
|
||||
page_size = key_cache.shape[1]
|
||||
num_pages = (kv_len + page_size - 1) // page_size
|
||||
page_ids = block_table[req, :num_pages].long()
|
||||
k = key_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
v = value_cache[page_ids].reshape(-1, num_kv_heads, HEAD_DIM)[:kv_len]
|
||||
k = k.float().repeat_interleave(gqa_group, dim=1)
|
||||
v = v.float().repeat_interleave(gqa_group, dim=1)
|
||||
|
||||
q_req = q[q_start : q_start + q_len].float()
|
||||
rel_req = rel_logits[q_start : q_start + q_len].float()
|
||||
scores = torch.einsum("qhd,khd->hqk", q_req, k) / HEAD_DIM
|
||||
|
||||
q_pos = torch.arange(q_len, device=q.device)[:, None] + kv_len - q_len
|
||||
k_pos = torch.arange(kv_len, device=q.device)[None, :]
|
||||
distance = q_pos - k_pos
|
||||
rel_idx = distance.clamp(0, rel_extent - 1)
|
||||
bias = rel_req.permute(1, 0, 2).gather(
|
||||
2, rel_idx[None].expand(num_heads, -1, -1)
|
||||
)
|
||||
in_rel_extent = (distance >= 0) & (distance < rel_extent)
|
||||
scores += torch.where(in_rel_extent[None], bias, 0.0)
|
||||
|
||||
masked = distance < 0
|
||||
if window_left is not None:
|
||||
masked |= distance > window_left
|
||||
scores.masked_fill_(masked[None], float("-inf"))
|
||||
out.append(torch.einsum("hqk,khd->qhd", scores.softmax(-1), v))
|
||||
q_start += q_len
|
||||
|
||||
return torch.cat(out).to(DTYPE)
|
||||
|
||||
|
||||
def test_query_length_bucket():
|
||||
assert bucket_max_seqlen_q(1) == 1
|
||||
assert bucket_max_seqlen_q(17) == 32
|
||||
assert bucket_max_seqlen_q(33) == 64
|
||||
|
||||
|
||||
def test_split_kv_dispatch_thresholds():
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=4,
|
||||
max_kv_len=65536,
|
||||
page_size=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=4096,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
window_left=-1,
|
||||
)
|
||||
assert use_split_kv_decode(
|
||||
max_query_len=1,
|
||||
max_kv_len=512,
|
||||
page_size=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert decode_split_count(512, 511) == 4
|
||||
assert decode_split_count(65536, -1) == 32
|
||||
|
||||
|
||||
def test_gfx950_gluon_dispatch_is_strict(monkeypatch: pytest.MonkeyPatch):
|
||||
import vllm.models.inkling.amd.ops.fa4_rel_attention as rel_attention
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=16, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=4, page_size=128, head_dim=128)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=256)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: False)
|
||||
assert not use_gfx950_gluon_decode(max_query_len=1, page_size=128, head_dim=128)
|
||||
|
||||
monkeypatch.setattr(rel_attention, "on_gfx950", lambda: True)
|
||||
assert use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=128,
|
||||
head_dim=128,
|
||||
window_left=511,
|
||||
)
|
||||
assert not use_gfx950_gluon_extend(
|
||||
max_query_len=4,
|
||||
max_kv_len=8192,
|
||||
page_size=16,
|
||||
head_dim=128,
|
||||
window_left=-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "window_left", "max_kv_len"),
|
||||
[
|
||||
(16, None, None),
|
||||
(16, 15, None),
|
||||
(128, None, 8192),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_ragged_multi_page_relative_attention(
|
||||
page_size: int,
|
||||
window_left: int | None,
|
||||
max_kv_len: int | None,
|
||||
):
|
||||
"""Covers full/chunked prefill, decode, GQA, and the local window."""
|
||||
torch.manual_seed(19 + int(window_left is not None))
|
||||
device = "cuda"
|
||||
q_lens = [17, 1]
|
||||
kv_lens = [35, 80]
|
||||
num_heads = 8
|
||||
num_kv_heads = 2
|
||||
rel_extent = 16
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(sum(q_lens), num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
1 + len(q_lens) * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
sum(q_lens), num_heads, rel_extent, device=device, dtype=DTYPE
|
||||
)
|
||||
|
||||
block_table = torch.stack(
|
||||
[
|
||||
torch.arange(1 + req * max_pages, 1 + (req + 1) * max_pages)
|
||||
for req in range(len(q_lens))
|
||||
]
|
||||
).to(device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.tensor(
|
||||
[0, *torch.tensor(q_lens).cumsum(0).tolist()],
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
preallocated = torch.empty_like(q)
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=bucket_max_seqlen_q(max(q_lens)),
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
out=preallocated,
|
||||
)
|
||||
assert actual.data_ptr() == preallocated.data_ptr()
|
||||
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("page_size", "kv_lens", "num_kv_heads", "rel_extent", "window_left"),
|
||||
[
|
||||
(16, [257, 513], 1, 64, None),
|
||||
(128, [257, 512], 2, 512, 511),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_split_kv_decode_matches_reference(
|
||||
page_size: int,
|
||||
kv_lens: list[int],
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
):
|
||||
"""Covers page-16 split-KV and page-128 gfx950 Gluon decode."""
|
||||
torch.manual_seed(41 + page_size)
|
||||
device = "cuda"
|
||||
batch_size = len(kv_lens)
|
||||
q_lens = [1] * batch_size
|
||||
num_heads = 8
|
||||
max_pages = max((length + page_size - 1) // page_size for length in kv_lens)
|
||||
|
||||
q = torch.randn(batch_size, num_heads, HEAD_DIM, device=device)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
key_cache = torch.randn(
|
||||
batch_size * max_pages,
|
||||
page_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
rel_logits = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * max_pages,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, max_pages)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
batch_size + 1,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
max_kv_len = 8192 if page_size == 16 else max(kv_lens)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=1,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"q_len",
|
||||
"kv_len",
|
||||
"num_kv_heads",
|
||||
"rel_extent",
|
||||
"window_left",
|
||||
"gluon_enabled",
|
||||
),
|
||||
[
|
||||
(1, 513, 2, 512, 511, True),
|
||||
(1, 513, 2, 512, 511, False),
|
||||
(4, 8192, 1, 1024, None, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_gfx950_page128_packed_kv_views_match_reference(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
q_len: int,
|
||||
kv_len: int,
|
||||
num_kv_heads: int,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
gluon_enabled: bool,
|
||||
):
|
||||
"""Cover both Gluon and split-KV against vLLM's packed KV allocation."""
|
||||
monkeypatch.setenv(
|
||||
"INKLING_GFX950_GLUON",
|
||||
"1" if gluon_enabled else "0",
|
||||
)
|
||||
torch.manual_seed(71 + q_len)
|
||||
device = "cuda"
|
||||
page_size = 128
|
||||
batch_size = 2
|
||||
num_heads = 8
|
||||
q_lens = [q_len] * batch_size
|
||||
kv_lens = [kv_len] * batch_size
|
||||
pages_per_req = (kv_len + page_size - 1) // page_size
|
||||
|
||||
q = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
|
||||
# FlashAttentionBackend allocates logical [block, head, page, 2 * dim].
|
||||
# Inkling transposes to [block, page, head, 2 * dim] and splits K/V,
|
||||
# producing non-contiguous views whose page/head strides include both.
|
||||
packed_kv = torch.randn(
|
||||
batch_size * pages_per_req,
|
||||
num_kv_heads,
|
||||
page_size,
|
||||
2 * HEAD_DIM,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
key_cache, value_cache = packed_kv.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
assert not key_cache.is_contiguous()
|
||||
assert key_cache.stride() == value_cache.stride()
|
||||
|
||||
rel_logits = torch.randn(
|
||||
batch_size * q_len,
|
||||
num_heads,
|
||||
rel_extent,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * pages_per_req,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
).view(batch_size, pages_per_req)
|
||||
cache_seqlens = torch.tensor(kv_lens, device=device, dtype=torch.int32)
|
||||
cu_seqlens_q = torch.arange(
|
||||
0,
|
||||
batch_size * q_len + 1,
|
||||
q_len,
|
||||
device=device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
actual = inkling_fa4_rel_attention(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=q_len,
|
||||
softmax_scale=1.0 / HEAD_DIM,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
max_kv_len=kv_len,
|
||||
)
|
||||
expected = _reference(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
block_table,
|
||||
q_lens,
|
||||
kv_lens,
|
||||
rel_extent,
|
||||
window_left,
|
||||
)
|
||||
torch.testing.assert_close(actual.float(), expected.float(), atol=3e-2, rtol=3e-2)
|
||||
@@ -1,116 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm numerical-equivalence tests for the fused MTP depth-layer input kernel.
|
||||
|
||||
``embed_dual_rmsnorm_cat`` must match the unfused module sequence numerically:
|
||||
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
|
||||
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
|
||||
chained backbone embed_norm and the depth embed_norm), and the fused row
|
||||
gather matches ``F.embedding``.
|
||||
"""
|
||||
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.amd.ops.norm import (
|
||||
embed_dual_rmsnorm_cat,
|
||||
embed_rmsnorm,
|
||||
rmsnorm,
|
||||
)
|
||||
|
||||
EPS = 1e-6
|
||||
VOCAB = 4096
|
||||
BF16_ATOL = torch.finfo(torch.bfloat16).eps
|
||||
|
||||
|
||||
def _assert_bf16_close(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# Different Triton reduction schedules can move a small number of outputs
|
||||
# by one bf16 epsilon while remaining numerically equivalent.
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=BF16_ATOL)
|
||||
|
||||
|
||||
def _ref(hidden, w_h, w_e, emb, w_pre=None):
|
||||
if w_pre is not None:
|
||||
emb = rmsnorm(emb, w_pre, EPS)
|
||||
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
|
||||
emb = table[ids.long()]
|
||||
|
||||
# Fused gather + chained backbone pre-norm (the decode draft-step path).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden,
|
||||
w_h,
|
||||
w_e,
|
||||
EPS,
|
||||
input_ids=ids,
|
||||
embed_table=table,
|
||||
pre_norm_weight=w_pre,
|
||||
)
|
||||
assert out.shape == (t, 2 * n)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb, w_pre))
|
||||
|
||||
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
|
||||
# embeddings, already backbone-normed).
|
||||
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
# Fused gather, no pre-norm (use_embed_norm=False).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
|
||||
)
|
||||
_assert_bf16_close(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
ref_emb = table[ids.long()]
|
||||
|
||||
# Gather + embed_norm (base model / MTP prefill embed path).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, w, EPS))
|
||||
assert out.shape == (t, n)
|
||||
_assert_bf16_close(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
|
||||
|
||||
# Pure gather (use_embed_norm=False / replicated module forward).
|
||||
out = cast(torch.Tensor, embed_rmsnorm(ids, table, None, EPS))
|
||||
assert torch.equal(out, ref_emb)
|
||||
|
||||
# Chained first-layer attn_norm (the target text-path forward): one launch
|
||||
# emits both the residual and layer 0's normed attention input.
|
||||
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain),
|
||||
)
|
||||
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
|
||||
_assert_bf16_close(res, ref_res)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
|
||||
|
||||
# Chained without embed_norm (use_embed_norm=False).
|
||||
res, attn_in = cast(
|
||||
tuple[torch.Tensor, torch.Tensor],
|
||||
embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain),
|
||||
)
|
||||
assert torch.equal(res, ref_emb)
|
||||
_assert_bf16_close(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
|
||||
@@ -1,20 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.inkling.amd.sconv_swa_attn import InklingConvState
|
||||
|
||||
|
||||
def test_runtime_sconv_block_size_tracks_unified_cache_page():
|
||||
"""The cache planner may enlarge W=4 blocks to match attention pages."""
|
||||
owner = InklingConvState.__new__(InklingConvState)
|
||||
nn.Module.__init__(owner)
|
||||
owner.block_size = 4
|
||||
|
||||
owner.kv_cache = torch.tensor([])
|
||||
assert owner.cache_block_size == 4
|
||||
|
||||
owner.kv_cache = torch.empty(2, 1, 32, 1024)
|
||||
assert owner.cache_block_size == 32
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
|
||||
from vllm.models.inkling.common.mm_preprocess import InklingMultiModalDataParser
|
||||
from vllm.models.inkling.common.towers import plan_out_scales
|
||||
from vllm.models.inkling.configs import (
|
||||
InklingAudioConfig,
|
||||
InklingModelConfig,
|
||||
InklingVisionConfig,
|
||||
)
|
||||
from vllm.models.inkling.nvidia.sconv_swa_attn import (
|
||||
InklingSconvMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
|
||||
|
||||
def test_vision_scale_plan_matches_released_config():
|
||||
assert plan_out_scales(2, 40, 4) == [
|
||||
(1, 1, 1, 3),
|
||||
(1, 5, 5, 128),
|
||||
(1, 10, 10, 320),
|
||||
(1, 40, 40, 4800),
|
||||
(2, 40, 40, 9600),
|
||||
]
|
||||
|
||||
|
||||
def test_vision_scale_plan_breaks_assignment_ties_in_order():
|
||||
reductions = [np.prod(scale[:-1]) for scale in plan_out_scales(2, 52, 4)]
|
||||
|
||||
assert reductions == sorted(set(reductions))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config_cls", "kwargs", "missing"),
|
||||
[
|
||||
(InklingAudioConfig, {"decoder_dmodel": 16}, "n_mel_bins"),
|
||||
(InklingVisionConfig, {"decoder_dmodel": 16}, "vision_encoder_type"),
|
||||
],
|
||||
)
|
||||
def test_enabled_tower_requires_architecture_fields(config_cls, kwargs, missing):
|
||||
with pytest.raises(ValueError, match=missing):
|
||||
config_cls(**kwargs)
|
||||
|
||||
|
||||
def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
|
||||
parser = InklingMultiModalDataParser(target_sr=16_000, target_channels=1)
|
||||
with pytest.raises(ValueError, match="ambiguous channel layout"):
|
||||
parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
|
||||
|
||||
|
||||
def test_inkling_mtp_chain_norm_is_disabled_by_default():
|
||||
assert InklingModelConfig().chain_hidden_post_norm is False
|
||||
|
||||
|
||||
def test_inkling_supports_piecewise_cudagraphs():
|
||||
support = InklingSconvMetadataBuilder.get_cudagraph_support
|
||||
assert support(None, None) == AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
compilation_config = CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||
splitting_ops=[],
|
||||
)
|
||||
resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
|
||||
AttentionCGSupport.UNIFORM_BATCH,
|
||||
"InklingSconvBackend",
|
||||
)
|
||||
|
||||
assert resolved_mode == CUDAGraphMode.PIECEWISE
|
||||
@@ -1,428 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness test for the Inkling FA4 relative-attention score-mod kernel.
|
||||
|
||||
Checks ``_INKLING_FA4_REL_ATTENTION_KERNEL`` against a pure-PyTorch reference
|
||||
that implements the relative bias exactly as documented in the Inkling architecture
|
||||
guide::
|
||||
|
||||
logit(i, j, h) = (1 / head_dim) * dot(q[i, h], k[j, h]) + rel_bias(i, j, h)
|
||||
rel_bias(i, j, h) = rel_logits[i, h, i - j] if 0 <= i - j < rel_extent
|
||||
= 0 otherwise
|
||||
|
||||
with causal (and optionally sliding-window) masking handled by the backend.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.nvidia.attention import (
|
||||
InklingAttention,
|
||||
compute_log_scaling_tau,
|
||||
)
|
||||
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
|
||||
_INKLING_FA4_REL_ATTENTION_KERNEL,
|
||||
_use_sheared_bias,
|
||||
inkling_fa4_num_splits,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
|
||||
_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
|
||||
|
||||
NUM_HEADS = [(4, 4), (8, 2)] # (num_heads, num_kv_heads)
|
||||
GLOBAL_REL_EXTENTS = [128, 1024]
|
||||
LOCAL_REL_EXTENTS = [128, 256]
|
||||
HEAD_DIM = 128
|
||||
BLOCK_SIZE = 16
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def test_log_scaling_tau_matches_reference():
|
||||
positions = torch.tensor([0, 127999, 128000, 999999], dtype=torch.int64)
|
||||
actual = compute_log_scaling_tau(positions, 128000, 0.1)
|
||||
expected = 1.0 + 0.1 * torch.log(
|
||||
torch.clamp((positions + 1).float() / 128000.0, min=1.0)
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_split_packed_kv_cache():
|
||||
attention = InklingAttention.__new__(InklingAttention)
|
||||
torch.nn.Module.__init__(attention)
|
||||
attention.head_dim = 8
|
||||
attention.kv_cache = torch.arange(2 * 3 * 4 * 16).reshape(2, 3, 4, 16)
|
||||
|
||||
key_cache, value_cache = attention._split_kv_cache()
|
||||
|
||||
assert key_cache.shape == value_cache.shape == (2, 4, 3, 8)
|
||||
torch.testing.assert_close(key_cache, attention.kv_cache[..., :8].transpose(1, 2))
|
||||
torch.testing.assert_close(value_cache, attention.kv_cache[..., 8:].transpose(1, 2))
|
||||
|
||||
|
||||
def test_num_splits_hopper_is_unsplit(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(major=9, minor=0),
|
||||
)
|
||||
assert (
|
||||
inkling_fa4_num_splits(
|
||||
is_local=False,
|
||||
batch_size=1,
|
||||
max_query_len=1,
|
||||
num_heads=16,
|
||||
num_kv_heads=2,
|
||||
max_kv_len=1_048_576,
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "expected"),
|
||||
[(9, False), (10, True), (11, True), (12, False)],
|
||||
)
|
||||
def test_sheared_bias_architecture_selection(monkeypatch, major, expected):
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(major=major, minor=0),
|
||||
)
|
||||
_use_sheared_bias.cache_clear()
|
||||
try:
|
||||
assert _use_sheared_bias() is expected
|
||||
finally:
|
||||
_use_sheared_bias.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def blackwell_platform(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"get_device_capability",
|
||||
lambda: DeviceCapability(major=10, minor=0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size", "max_query_len", "expected"),
|
||||
[
|
||||
(1, 1, (16, 32, 128, 128)),
|
||||
(8, 1, (2, 4, 8, 16)),
|
||||
(32, 1, (1, 1, 2, 4)),
|
||||
(1, 128, (2, 4, 8, 16)),
|
||||
(1, 2048, (1, 1, 1, 1)),
|
||||
],
|
||||
)
|
||||
def test_num_splits_all_tp(blackwell_platform, batch_size, max_query_len, expected):
|
||||
actual = tuple(
|
||||
inkling_fa4_num_splits(
|
||||
is_local=False,
|
||||
batch_size=batch_size,
|
||||
max_query_len=max_query_len,
|
||||
num_heads=64 // tp,
|
||||
num_kv_heads=8 // tp,
|
||||
max_kv_len=131072,
|
||||
)
|
||||
for tp in (1, 2, 4, 8)
|
||||
)
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp", [1, 2, 4, 8])
|
||||
def test_num_splits_local_is_unsplit(tp):
|
||||
assert (
|
||||
inkling_fa4_num_splits(
|
||||
is_local=True,
|
||||
batch_size=1,
|
||||
max_query_len=1,
|
||||
num_heads=64 // tp,
|
||||
num_kv_heads=16 // tp,
|
||||
max_kv_len=512,
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("max_kv_len", "expected"),
|
||||
[(8192, 32), (65536, 64), (1048576, 128)],
|
||||
)
|
||||
@pytest.mark.parametrize("tp", [4, 8])
|
||||
def test_num_splits_long_context_bound(blackwell_platform, tp, max_kv_len, expected):
|
||||
assert (
|
||||
inkling_fa4_num_splits(
|
||||
is_local=False,
|
||||
batch_size=1,
|
||||
max_query_len=1,
|
||||
num_heads=64 // tp,
|
||||
num_kv_heads=8 // tp,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def _ref_rel_attn(
|
||||
q: torch.Tensor, # [total_q, H, D]
|
||||
key_cache: torch.Tensor, # [num_blocks, block, Hkv, D]
|
||||
value_cache: torch.Tensor,
|
||||
rel_logits: torch.Tensor, # [total_q, H, rel_extent]
|
||||
*,
|
||||
q_lens: list[int],
|
||||
kv_lens: list[int],
|
||||
block_table: torch.Tensor,
|
||||
scale: float,
|
||||
rel_extent: int,
|
||||
window_left: int | None,
|
||||
) -> torch.Tensor:
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
num_heads = q.shape[1]
|
||||
g = num_heads // num_kv_heads
|
||||
bt = block_table.cpu().numpy()
|
||||
out = torch.empty_like(q)
|
||||
|
||||
start = 0
|
||||
for i, (ql, kl) in enumerate(zip(q_lens, kv_lens)):
|
||||
qi = q[start : start + ql].float() # [ql, H, D]
|
||||
rl = rel_logits[start : start + ql].float() # [ql, H, rel_extent]
|
||||
|
||||
nblk = (kl + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
blk = bt[i, :nblk]
|
||||
k = key_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
|
||||
v = value_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
|
||||
k = k.repeat_interleave(g, dim=1) # [kl, H, D]
|
||||
v = v.repeat_interleave(g, dim=1)
|
||||
|
||||
# [H, ql, kl]
|
||||
scores = torch.einsum("qhd,khd->hqk", qi, k) * scale
|
||||
|
||||
dev = q.device
|
||||
qpos = torch.arange(ql, device=dev).view(ql, 1) + (kl - ql) # query pos
|
||||
kpos = torch.arange(kl, device=dev).view(1, kl)
|
||||
dist = qpos - kpos # [ql, kl] = i - j
|
||||
|
||||
# Relative bias: rel_logits[i, h, dist] when 0 <= dist < rel_extent.
|
||||
in_rng = (dist >= 0) & (dist < rel_extent) # [ql, kl]
|
||||
idx = dist.clamp(0, rel_extent - 1)
|
||||
# gather per head: bias[h, i, j] = rl[i, h, idx[i, j]]
|
||||
bias = rl.permute(1, 0, 2).gather( # [H, ql, rel_extent]
|
||||
2, idx.unsqueeze(0).expand(num_heads, -1, -1)
|
||||
) # [H, ql, kl]
|
||||
bias = torch.where(in_rng.unsqueeze(0), bias, torch.zeros_like(bias))
|
||||
scores = scores + bias
|
||||
|
||||
mask = dist < 0 # causal
|
||||
if window_left is not None:
|
||||
mask = mask | (dist > window_left)
|
||||
scores.masked_fill_(mask.unsqueeze(0), float("-inf"))
|
||||
|
||||
probs = torch.softmax(scores, dim=-1)
|
||||
out[start : start + ql] = torch.einsum("hqk,khd->qhd", probs, v).to(q.dtype)
|
||||
start += ql
|
||||
return out
|
||||
|
||||
|
||||
def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0):
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
q_lens = [s[0] for s in seq_lens]
|
||||
kv_lens = [s[1] for s in seq_lens]
|
||||
total_q = sum(q_lens)
|
||||
num_seqs = len(seq_lens)
|
||||
|
||||
scale = 1.0 / HEAD_DIM
|
||||
|
||||
# q/k are RMS-normed in the model (unit-ish norm); normalize here so the
|
||||
# logit magnitudes are realistic and the bias is not numerically dwarfed.
|
||||
q = torch.randn(total_q, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
|
||||
q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
|
||||
|
||||
# Paged KV cache.
|
||||
max_blocks = (max(kv_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
num_blocks = num_seqs * max_blocks + 1
|
||||
key_cache = torch.randn(
|
||||
num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
|
||||
)
|
||||
key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
|
||||
value_cache = torch.randn(
|
||||
num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
|
||||
)
|
||||
|
||||
# Distinct blocks per sequence (block 0 left as a never-referenced pad).
|
||||
block_table = torch.zeros(num_seqs, max_blocks, dtype=torch.int32, device=device)
|
||||
for i in range(num_seqs):
|
||||
block_table[i] = torch.arange(
|
||||
1 + i * max_blocks, 1 + (i + 1) * max_blocks, dtype=torch.int32
|
||||
)
|
||||
|
||||
cu_seqlens_q = torch.tensor(
|
||||
[0, *torch.cumsum(torch.tensor(q_lens), 0).tolist()],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
cache_seqlens = torch.tensor(kv_lens, dtype=torch.int32, device=device)
|
||||
|
||||
rel_logits = torch.randn(total_q, num_heads, rel_extent, device=device, dtype=DTYPE)
|
||||
|
||||
window_size = (-1, -1) if window_left is None else (window_left, 0)
|
||||
|
||||
preallocated_out = torch.empty_like(q)
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=window_left is not None,
|
||||
batch_size=num_seqs,
|
||||
max_query_len=max(q_lens),
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
max_kv_len=max(kv_lens),
|
||||
)
|
||||
out = _INKLING_FA4_REL_ATTENTION_KERNEL(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
block_table=block_table,
|
||||
cache_seqlens=cache_seqlens,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=max(q_lens),
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
rel_extent=rel_extent,
|
||||
rel_logits=rel_logits,
|
||||
num_splits=num_splits,
|
||||
out=preallocated_out,
|
||||
)
|
||||
assert out.data_ptr() == preallocated_out.data_ptr()
|
||||
out = out.view(total_q, num_heads, HEAD_DIM)
|
||||
|
||||
ref = _ref_rel_attn(
|
||||
q,
|
||||
key_cache,
|
||||
value_cache,
|
||||
rel_logits,
|
||||
q_lens=q_lens,
|
||||
kv_lens=kv_lens,
|
||||
block_table=block_table,
|
||||
scale=scale,
|
||||
rel_extent=rel_extent,
|
||||
window_left=window_left,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out.float(), ref.float(), atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_score_mod_relative_attention(monkeypatch):
|
||||
module = importlib.import_module("vllm.models.inkling.nvidia.ops.fa4_rel_attention")
|
||||
monkeypatch.setattr(module, "_use_sheared_bias", lambda: False)
|
||||
_run_case(
|
||||
[(64, 64), (1, 80)],
|
||||
num_heads=4,
|
||||
num_kv_heads=4,
|
||||
rel_extent=128,
|
||||
window_left=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(64, 64)], # single full prefill
|
||||
[(64, 64), (33, 33), (17, 17)], # ragged prefill batch
|
||||
[(512, 512)], # seq_len >> rel_extent (most keys get zero bias)
|
||||
[(300, 300), (512, 512), (129, 129)], # large ragged batch
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
|
||||
@torch.inference_mode()
|
||||
def test_full_attention(seq_lens, num_heads, rel_extent):
|
||||
# rel_extent=128 exercises the out-of-range (zero bias) path; 1024 covers all.
|
||||
# With the 512-token cases and rel_extent=128, query/seq lengths are far
|
||||
# larger than rel_extent so the vast majority of (i, j) pairs are out of
|
||||
# range and must contribute zero bias.
|
||||
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(200, 512)], # chunked prefill: q_len=200 (> rel_extent), 312 cached
|
||||
[(200, 512), (50, 300), (1, 400)], # mixed chunked + decode
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
|
||||
@torch.inference_mode()
|
||||
def test_chunked_prefill(seq_lens, num_heads, rel_extent):
|
||||
# q_len < kv_len with q_len itself larger than rel_extent (for the 128 case):
|
||||
# exercises the seqlen_k - seqlen_q offset together with the out-of-range path.
|
||||
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(64, 64), (40, 40)], # seq_len > window
|
||||
[(512, 512), (300, 300)], # seq_len/query_len >> window
|
||||
[(1, 512)], # decode with kv_len >> window
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("local_extent", LOCAL_REL_EXTENTS)
|
||||
@torch.inference_mode()
|
||||
def test_sliding_window(seq_lens, num_heads, local_extent):
|
||||
# Local layers use window_size=(local_extent-1, 0) and rel_extent==local_extent.
|
||||
# With the 512-token cases, query/seq lengths far exceed the window so most
|
||||
# keys are masked out by the sliding window.
|
||||
_run_case(
|
||||
seq_lens,
|
||||
num_heads[0],
|
||||
num_heads[1],
|
||||
rel_extent=local_extent,
|
||||
window_left=local_extent - 1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.skipif(
|
||||
_cap is None or _cap.major < 9,
|
||||
reason="FA4 score-mod requires Hopper+ (SM90+)",
|
||||
)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens",
|
||||
[
|
||||
[(1, 50)],
|
||||
[(1, 50), (1, 7), (1, 200)],
|
||||
[(1, 512), (1, 333)], # kv_len >> rel_extent
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
|
||||
@torch.inference_mode()
|
||||
def test_decode(seq_lens, num_heads, rel_extent):
|
||||
# q_len=1 with kv_len>q_len: the score-mod's seqlen_k - seqlen_q offset path.
|
||||
_run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
|
||||
@@ -1,124 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.models.inkling.nvidia.ops.fa4_rel_attention as fa4_rel_attention
|
||||
from vllm.models.inkling.configs import InklingMMConfig, InklingModelConfig
|
||||
from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
|
||||
InklingFA4RelAttentionKernel,
|
||||
_num_warps_bucket,
|
||||
bucket_max_seqlen_q,
|
||||
inkling_fa4_num_splits,
|
||||
)
|
||||
|
||||
|
||||
def _vllm_config_from_reference_config(config: SimpleNamespace) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=InklingMMConfig(
|
||||
text_config=InklingModelConfig(
|
||||
num_attention_heads=config.num_heads,
|
||||
num_key_value_heads=config.num_kv_heads,
|
||||
head_dim=config.head_dim,
|
||||
rel_extent=config.rel_extent,
|
||||
swa_num_attention_heads=config.num_heads,
|
||||
swa_num_key_value_heads=config.num_kv_heads,
|
||||
swa_head_dim=config.head_dim,
|
||||
sliding_window_size=config.rel_extent,
|
||||
)
|
||||
),
|
||||
max_model_len=config.max_kv_len,
|
||||
dtype=config.dtype,
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
cache_dtype="auto",
|
||||
block_size=config.block_size,
|
||||
),
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_seqs=config.max_num_reqs,
|
||||
max_num_batched_tokens=config.max_num_batched_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_bucket_max_seqlen_q():
|
||||
assert [bucket_max_seqlen_q(n) for n in range(1, 10)] == [
|
||||
1,
|
||||
2,
|
||||
4,
|
||||
4,
|
||||
8,
|
||||
8,
|
||||
8,
|
||||
8,
|
||||
16,
|
||||
]
|
||||
|
||||
|
||||
def test_warmup_enumerates_every_runtime_compile_class(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
fa4_rel_attention,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
num_heads=16,
|
||||
num_kv_heads=2,
|
||||
head_dim=128,
|
||||
rel_extent=1024,
|
||||
window_size=(-1, -1),
|
||||
is_local=False,
|
||||
max_kv_len=65536,
|
||||
dtype=torch.bfloat16,
|
||||
kv_dtype=torch.bfloat16,
|
||||
block_size=16,
|
||||
max_num_reqs=64,
|
||||
max_num_batched_tokens=192,
|
||||
)
|
||||
vllm_config = _vllm_config_from_reference_config(config)
|
||||
kernel = InklingFA4RelAttentionKernel()
|
||||
warmed_keys = set(kernel.get_warmup_keys(vllm_config))
|
||||
expected_keys: set[InklingFA4RelAttentionKernel.CompileKey] = set()
|
||||
|
||||
for is_local in (False, True):
|
||||
max_kv_len = config.rel_extent if is_local else config.max_kv_len
|
||||
window_size = (config.rel_extent - 1, 0) if is_local else (-1, -1)
|
||||
for query_len in range(1, config.max_num_batched_tokens + 1):
|
||||
max_seqlen_q = bucket_max_seqlen_q(query_len)
|
||||
max_num_reqs = min(
|
||||
config.max_num_reqs,
|
||||
config.max_num_batched_tokens - query_len + 1,
|
||||
)
|
||||
for num_reqs in range(1, max_num_reqs + 1):
|
||||
num_splits = inkling_fa4_num_splits(
|
||||
is_local=is_local,
|
||||
batch_size=num_reqs,
|
||||
max_query_len=max_seqlen_q,
|
||||
num_heads=config.num_heads,
|
||||
num_kv_heads=config.num_kv_heads,
|
||||
max_kv_len=max_kv_len,
|
||||
)
|
||||
expected_keys.add(
|
||||
InklingFA4RelAttentionKernel.CompileKey(
|
||||
is_local=is_local,
|
||||
num_heads=config.num_heads,
|
||||
num_kv_heads=config.num_kv_heads,
|
||||
head_dim=config.head_dim,
|
||||
rel_extent=config.rel_extent,
|
||||
dtype=config.dtype,
|
||||
kv_dtype=config.kv_dtype,
|
||||
block_size=config.block_size,
|
||||
window_size=window_size,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
num_splits=num_splits,
|
||||
num_warps_bucket=(
|
||||
_num_warps_bucket(num_reqs) if num_splits > 1 else None
|
||||
),
|
||||
large_num_reqs=num_reqs > 1024,
|
||||
)
|
||||
)
|
||||
|
||||
assert warmed_keys == expected_keys
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused Inkling vision/audio tower kernels.
|
||||
|
||||
Each kernel is checked against a pure-PyTorch reference implementing the
|
||||
towers' original op sequence (native ``rms_norm`` semantics: fp32 variance,
|
||||
bf16-rounded weight multiply; exact-erf GELU; fp32-accumulated embedding sum).
|
||||
The fused kernels keep the same accumulation dtype and rounding points, so
|
||||
outputs differ from the reference only by reduction order — a few bf16 ulps.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("requires CUDA", allow_module_level=True)
|
||||
|
||||
from vllm.models.inkling.common.towers import fold_timespace_to_depth
|
||||
from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm, rmsnorm_gelu
|
||||
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _bf16_spacing(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.exp2(torch.floor(torch.log2(x.float().abs().clamp(min=1e-30)))) * (
|
||||
2**-7
|
||||
)
|
||||
|
||||
|
||||
def _assert_close_ulps(
|
||||
got: torch.Tensor,
|
||||
ref: torch.Tensor,
|
||||
max_ulps: float = 4.0,
|
||||
atol: float = 1e-3,
|
||||
pre_act: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""Fused vs reference differ only by fp32 reduction order, which shows up
|
||||
as a few bf16 ulps at any magnitude — a fixed rtol misrepresents that, so
|
||||
compare against the reference's local ulp spacing. ``pre_act`` (the
|
||||
reference pre-activation) adds derivative-propagated slack: near
|
||||
``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
|
||||
many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
|
||||
g, r = got.float(), ref.float()
|
||||
tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
|
||||
if pre_act is not None:
|
||||
tol = tol + 2.5 * _bf16_spacing(pre_act)
|
||||
bad = (g - r).abs() > tol
|
||||
assert not bad.any(), (
|
||||
f"{int(bad.sum())}/{ref.numel()} elements beyond tolerance; "
|
||||
f"max abs diff {(g - r).abs().max().item():.3e}"
|
||||
)
|
||||
|
||||
|
||||
def _ref_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
# Matches ir.ops.rms_norm: fp32 normalize, bf16 weight multiply.
|
||||
x32 = x.float()
|
||||
var = x32.pow(2).mean(dim=-1, keepdim=True)
|
||||
xn = x32 * torch.rsqrt(var + eps)
|
||||
return (xn.to(weight.dtype) * weight).to(x.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rows", [1, 7, 1200, 38400])
|
||||
@pytest.mark.parametrize("dim", [128, 320, 4800, 6144])
|
||||
@pytest.mark.parametrize("gelu", [True, False])
|
||||
def test_rmsnorm_gelu(rows: int, dim: int, gelu: bool) -> None:
|
||||
torch.manual_seed(rows * dim)
|
||||
x = torch.randn(rows, dim, device="cuda", dtype=DTYPE)
|
||||
w = torch.randn(dim, device="cuda", dtype=DTYPE)
|
||||
|
||||
h = _ref_rms_norm(x, w, 1e-5)
|
||||
ref = F.gelu(h) if gelu else h
|
||||
got = rmsnorm_gelu(x, w, 1e-5, gelu=gelu)
|
||||
|
||||
_assert_close_ulps(got, ref, pre_act=h if gelu else None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 5, 64])
|
||||
@pytest.mark.parametrize(
|
||||
"shape,fold",
|
||||
[
|
||||
((2, 8, 8, 128), (1, 2)), # the real L0 -> L1 vision transition
|
||||
((2, 4, 4, 320), (1, 2)),
|
||||
((2, 8, 8, 128), (2, 2)), # temporal + spatial fold
|
||||
],
|
||||
)
|
||||
def test_rmsnorm_gelu_folded_store(
|
||||
n: int, shape: tuple[int, ...], fold: tuple[int, int]
|
||||
) -> None:
|
||||
torch.manual_seed(n)
|
||||
x = torch.randn(n, *shape, device="cuda", dtype=DTYPE)
|
||||
w = torch.randn(shape[-1], device="cuda", dtype=DTYPE)
|
||||
|
||||
plain = rmsnorm_gelu(x, w, 1e-5, gelu=True)
|
||||
ref = fold_timespace_to_depth(plain, *fold)
|
||||
got = rmsnorm_gelu(x, w, 1e-5, gelu=True, fold=fold)
|
||||
|
||||
# The folded store is a pure permutation of the unfolded output.
|
||||
assert got.shape == ref.shape
|
||||
torch.testing.assert_close(got, ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_frames", [1, 3, 100, 4097])
|
||||
@pytest.mark.parametrize("with_norm", [True, False])
|
||||
def test_dmel_embed_sum_norm(num_frames: int, with_norm: bool) -> None:
|
||||
torch.manual_seed(num_frames)
|
||||
n_bins, vocab, dim = 80, 16, 6144
|
||||
idx = torch.randint(
|
||||
0, vocab, (num_frames, n_bins), device="cuda", dtype=torch.int32
|
||||
)
|
||||
table = torch.randn(n_bins * vocab, dim, device="cuda", dtype=DTYPE)
|
||||
norm_w = torch.randn(dim, device="cuda", dtype=DTYPE)
|
||||
|
||||
flat = (torch.arange(n_bins, device="cuda", dtype=torch.int32) * vocab).unsqueeze(
|
||||
0
|
||||
) + idx
|
||||
ref = (
|
||||
F.embedding(flat.reshape(-1).long(), table)
|
||||
.reshape(num_frames, n_bins, dim)
|
||||
.sum(dim=1)
|
||||
)
|
||||
if with_norm:
|
||||
ref = _ref_rms_norm(ref, norm_w, 1e-6)
|
||||
|
||||
got = dmel_embed_sum_norm(idx, table, norm_w if with_norm else None, 1e-6)
|
||||
_assert_close_ulps(got, ref)
|
||||
|
||||
|
||||
def test_empty_inputs() -> None:
|
||||
x = torch.empty(0, 320, device="cuda", dtype=DTYPE)
|
||||
w = torch.randn(320, device="cuda", dtype=DTYPE)
|
||||
assert rmsnorm_gelu(x, w, 1e-5).shape == (0, 320)
|
||||
|
||||
idx = torch.empty(0, 80, device="cuda", dtype=torch.int32)
|
||||
table = torch.randn(1280, 6144, device="cuda", dtype=DTYPE)
|
||||
assert dmel_embed_sum_norm(idx, table, None, 0.0).shape == (0, 6144)
|
||||
@@ -1,249 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
|
||||
from vllm.models.inkling.nvidia import moe
|
||||
from vllm.models.inkling.nvidia.model import _TmlForCausalLMBase
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_gate_loads_directly_into_padded_runtime_weight() -> None:
|
||||
gate = moe.InklingGate(
|
||||
d_model=4,
|
||||
n_routed_experts=5,
|
||||
n_shared_experts=2,
|
||||
experts_per_token=2,
|
||||
route_scale=1.0,
|
||||
)
|
||||
loaded = torch.arange(28, dtype=gate.weight.dtype).reshape(7, 4)
|
||||
|
||||
gate.weight.weight_loader(gate.weight, loaded)
|
||||
|
||||
assert gate.weight.shape == (8, 4)
|
||||
torch.testing.assert_close(gate.weight[:7], loaded)
|
||||
torch.testing.assert_close(gate.weight[7], torch.zeros(4))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(("num_tokens", "expected_calls"), [(1, 1), (64, 1), (65, 0)])
|
||||
def test_gate_uses_ll_bf16_gemm_through_token_limit(
|
||||
monkeypatch, num_tokens, expected_calls
|
||||
) -> None:
|
||||
gate = moe.InklingGate(
|
||||
d_model=8,
|
||||
n_routed_experts=5,
|
||||
n_shared_experts=2,
|
||||
experts_per_token=2,
|
||||
route_scale=1.0,
|
||||
).to(device="cuda", dtype=torch.bfloat16)
|
||||
hidden_states = torch.randn(num_tokens, 8, device="cuda", dtype=torch.bfloat16)
|
||||
calls = []
|
||||
|
||||
def fake_ll_bf16_gemm(x, weight):
|
||||
calls.append((x, weight))
|
||||
return torch.ones(num_tokens, 8, device="cuda", dtype=torch.float32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
moe.current_platform, "has_device_capability", lambda capability: True
|
||||
)
|
||||
monkeypatch.setattr(moe.ll_bf16, "is_available", lambda: True)
|
||||
monkeypatch.setattr(moe.ll_bf16, "ll_bf16_gemm", fake_ll_bf16_gemm)
|
||||
|
||||
logits = gate.compute_logits(hidden_states)
|
||||
|
||||
assert len(calls) == expected_calls
|
||||
if calls:
|
||||
assert calls[0][0] is hidden_states
|
||||
assert calls[0][1] is gate.weight
|
||||
assert logits.shape == (num_tokens, 8)
|
||||
assert logits.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.gate = moe.InklingGate(
|
||||
d_model=4,
|
||||
n_routed_experts=5,
|
||||
n_shared_experts=2,
|
||||
experts_per_token=2,
|
||||
route_scale=1.0,
|
||||
)
|
||||
|
||||
assert "gate" not in get_supported_lora_modules(model)
|
||||
|
||||
|
||||
def test_custom_embedding_is_not_a_lora_target() -> None:
|
||||
model = torch.nn.Module()
|
||||
model.embedding_modules = _TmlForCausalLMBase.embedding_modules
|
||||
|
||||
supported = get_supported_lora_modules(model)
|
||||
|
||||
assert "embed_tokens" not in supported
|
||||
assert "lm_head" in supported
|
||||
|
||||
|
||||
def test_inkling_mapper_maps_modelopt_exclusions() -> None:
|
||||
quant_config = ModelOptNvFp4Config.from_config(
|
||||
{
|
||||
"quantization": {
|
||||
"quant_algo": "NVFP4",
|
||||
"group_size": 16,
|
||||
"kv_cache_quant_algo": None,
|
||||
"exclude_modules": [
|
||||
"model.llm.layers.2.mlp.experts",
|
||||
"model.llm.layers.2.mlp.shared_experts",
|
||||
],
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
quant_config.apply_vllm_mapper(
|
||||
_TmlForCausalLMBase.hf_to_vllm_mapper.get_rename_mapper()
|
||||
)
|
||||
|
||||
assert quant_config.is_layer_excluded("model.layers.2.mlp.experts")
|
||||
assert quant_config.is_layer_excluded("model.layers.2.mlp.shared_experts")
|
||||
assert not quant_config.is_layer_excluded("model.layers.3.mlp.experts")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("nested", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"suffix",
|
||||
[
|
||||
"input_global_scale",
|
||||
"weight_global_scale",
|
||||
"weight_packed",
|
||||
"weight_scale",
|
||||
],
|
||||
)
|
||||
def test_inkling_mapper_maps_compressed_tensors_expert_params(
|
||||
projection: str, suffix: str, nested: bool
|
||||
) -> None:
|
||||
projection_param = (
|
||||
f"{projection}_weight.{suffix}" if nested else f"{projection}_{suffix}"
|
||||
)
|
||||
source = f"model.llm.layers.2.mlp.experts.{projection_param}"
|
||||
|
||||
mapped = _TmlForCausalLMBase.hf_to_vllm_mapper.apply_list([source])
|
||||
|
||||
assert mapped == [f"model.layers.2.mlp.experts.{projection}_{suffix}"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
|
||||
def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
|
||||
experts = SimpleNamespace(
|
||||
w13_input_scale=torch.nn.Parameter(torch.empty(3, 2)),
|
||||
w2_input_scale=torch.nn.Parameter(torch.empty(3)),
|
||||
)
|
||||
layer = SimpleNamespace(experts=SimpleNamespace(routed_experts=experts))
|
||||
|
||||
loaded = moe.InklingMoE.load_expert_weight(
|
||||
layer,
|
||||
f"experts.{projection}_weight.input_amax",
|
||||
torch.tensor([amax]),
|
||||
)
|
||||
|
||||
scale = getattr(experts, f"{projection}_input_scale")
|
||||
expected = torch.full_like(scale, amax / (448.0 * 6.0))
|
||||
torch.testing.assert_close(scale, expected)
|
||||
assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("projection", ["w13", "w2"])
|
||||
@pytest.mark.parametrize("scale_kind", ["input", "weight"])
|
||||
def test_moe_loads_compressed_tensors_global_scale(
|
||||
projection: str, scale_kind: str
|
||||
) -> None:
|
||||
param = torch.nn.Parameter(torch.empty(3, 2 if projection == "w13" else 1))
|
||||
if projection == "w2":
|
||||
param = torch.nn.Parameter(param.squeeze(1))
|
||||
experts = SimpleNamespace(
|
||||
**{f"{projection}_{scale_kind}_global_scale": param},
|
||||
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=0)),
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
experts=SimpleNamespace(routed_experts=experts),
|
||||
_local_expert_slots=lambda: {0: 0, 1: 1, 2: 2},
|
||||
)
|
||||
checkpoint_scale = torch.tensor([[1.0], [2.0], [3.0]])
|
||||
|
||||
loaded = moe.InklingMoE.load_expert_weight(
|
||||
layer,
|
||||
f"experts.{projection}_{scale_kind}_global_scale",
|
||||
checkpoint_scale,
|
||||
)
|
||||
|
||||
expected = (
|
||||
checkpoint_scale.expand_as(param) if param.ndim == 2 else checkpoint_scale[:, 0]
|
||||
)
|
||||
torch.testing.assert_close(param, expected)
|
||||
assert loaded == [f"experts.routed_experts.{projection}_{scale_kind}_global_scale"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("projection", "checkpoint_rows"), [("w13", 8), ("w2", 4)])
|
||||
def test_moe_loads_channelwise_scale_for_tp(
|
||||
projection: str, checkpoint_rows: int
|
||||
) -> None:
|
||||
param = torch.nn.Parameter(torch.empty(2, 4, 1))
|
||||
experts = SimpleNamespace(
|
||||
**{f"{projection}_weight_scale": param},
|
||||
moe_config=SimpleNamespace(moe_parallel_config=SimpleNamespace(tp_rank=1)),
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
experts=SimpleNamespace(routed_experts=experts),
|
||||
_local_expert_slots=lambda: {0: 0, 2: 1},
|
||||
)
|
||||
checkpoint_scale = torch.arange(3 * checkpoint_rows).reshape(3, checkpoint_rows, 1)
|
||||
|
||||
loaded = moe.InklingMoE.load_expert_weight(
|
||||
layer, f"experts.{projection}_weight_scale", checkpoint_scale
|
||||
)
|
||||
|
||||
expected = checkpoint_scale[[0, 2]]
|
||||
if projection == "w13":
|
||||
expected = expected[:, 4:].reshape(2, 2, 2, 1).transpose(1, 2).flatten(1, 2)
|
||||
torch.testing.assert_close(param, expected.float())
|
||||
assert loaded == [f"experts.routed_experts.{projection}_weight_scale"]
|
||||
|
||||
|
||||
def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8)
|
||||
loaded = torch.arange(48, dtype=sink.w2_weight.dtype).reshape(2, 3, 8)
|
||||
|
||||
sink.load_weight("w2_weight", loaded)
|
||||
|
||||
expected = loaded[:, :, 4:].permute(1, 0, 2).reshape(3, 8)
|
||||
torch.testing.assert_close(sink.w2_weight, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
def test_sink_packed_weight_forward_matches_expert_sum(monkeypatch) -> None:
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8).to(
|
||||
device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
torch.manual_seed(1)
|
||||
w13 = torch.randn(2, 16, 3, dtype=torch.bfloat16)
|
||||
w2 = torch.randn(2, 3, 8, dtype=torch.bfloat16)
|
||||
sink.load_weight("w13_weight", w13)
|
||||
sink.load_weight("w2_weight", w2)
|
||||
x = torch.randn(5, 3, device="cuda", dtype=torch.bfloat16)
|
||||
gammas = torch.randn(5, 2, device="cuda")
|
||||
|
||||
output = sink(x, gammas)
|
||||
|
||||
raw = torch.einsum("td,efd->tef", x, w13[:, 8:].to("cuda"))
|
||||
hidden = torch.nn.functional.silu(raw[:, :, 0::2].float())
|
||||
hidden = (hidden * raw[:, :, 1::2] * gammas[:, :, None]).to(torch.bfloat16)
|
||||
expected = torch.einsum("tef,edf->td", hidden, w2[:, :, 4:].to("cuda"))
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Bit-exactness tests for the fused MTP depth-layer input kernel.
|
||||
|
||||
``embed_dual_rmsnorm_cat`` must match the unfused module sequence exactly:
|
||||
each rmsnorm computes in fp32 and rounds to bf16 at the same points as the
|
||||
vendored ``rmsnorm`` kernel (including the bf16 round-trip between the
|
||||
chained backbone embed_norm and the depth embed_norm), and the fused row
|
||||
gather matches ``F.embedding``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("requires CUDA", allow_module_level=True)
|
||||
|
||||
from vllm.models.inkling.nvidia.ops.norm import (
|
||||
embed_dual_rmsnorm_cat,
|
||||
embed_rmsnorm,
|
||||
rmsnorm,
|
||||
)
|
||||
|
||||
EPS = 1e-6
|
||||
VOCAB = 4096
|
||||
|
||||
|
||||
def _ref(hidden, w_h, w_e, emb, w_pre=None):
|
||||
if w_pre is not None:
|
||||
emb = rmsnorm(emb, w_pre, EPS)
|
||||
return torch.cat([rmsnorm(hidden, w_h, EPS), rmsnorm(emb, w_e, EPS)], dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_dual_rmsnorm_cat(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w_h = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
w_e = (1 + 0.01 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
w_pre = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
hidden = (torch.randn(t, n, device=dev) * 2).to(torch.bfloat16)
|
||||
emb = table[ids.long()]
|
||||
|
||||
# Fused gather + chained backbone pre-norm (the decode draft-step path).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden,
|
||||
w_h,
|
||||
w_e,
|
||||
EPS,
|
||||
input_ids=ids,
|
||||
embed_table=table,
|
||||
pre_norm_weight=w_pre,
|
||||
)
|
||||
assert out.shape == (t, 2 * n)
|
||||
assert torch.equal(out, _ref(hidden, w_h, w_e, emb, w_pre))
|
||||
|
||||
# Precomputed embeds, no pre-norm (draft prefill with target-merged MM
|
||||
# embeddings, already backbone-normed).
|
||||
out = embed_dual_rmsnorm_cat(hidden, w_h, w_e, EPS, embeds=emb)
|
||||
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
# Fused gather, no pre-norm (use_embed_norm=False).
|
||||
out = embed_dual_rmsnorm_cat(
|
||||
hidden, w_h, w_e, EPS, input_ids=ids, embed_table=table
|
||||
)
|
||||
assert torch.equal(out, _ref(hidden, w_h, w_e, emb))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1536, 6144])
|
||||
@pytest.mark.parametrize("t", [0, 1, 7, 256])
|
||||
@pytest.mark.parametrize("ids_dtype", [torch.int32, torch.int64])
|
||||
def test_embed_rmsnorm(n: int, t: int, ids_dtype: torch.dtype) -> None:
|
||||
torch.manual_seed(0)
|
||||
dev = "cuda"
|
||||
table = (torch.randn(VOCAB, n, device=dev) * 0.3).to(torch.bfloat16)
|
||||
w = torch.randn(n, device=dev).to(torch.bfloat16)
|
||||
ids = torch.randint(0, VOCAB, (t,), device=dev, dtype=ids_dtype)
|
||||
ref_emb = table[ids.long()]
|
||||
|
||||
# Gather + embed_norm (base model / MTP prefill embed path).
|
||||
out = embed_rmsnorm(ids, table, w, EPS)
|
||||
assert out.shape == (t, n)
|
||||
assert torch.equal(out, rmsnorm(ref_emb, w, EPS) if t else ref_emb)
|
||||
|
||||
# Pure gather (use_embed_norm=False / replicated module forward).
|
||||
out = embed_rmsnorm(ids, table, None, EPS)
|
||||
assert torch.equal(out, ref_emb)
|
||||
|
||||
# Chained first-layer attn_norm (the target text-path forward): one launch
|
||||
# emits both the residual and layer 0's normed attention input.
|
||||
w_chain = (1 + 0.05 * torch.randn(n, device=dev)).to(torch.bfloat16)
|
||||
res, attn_in = embed_rmsnorm(ids, table, w, EPS, chain_weight=w_chain)
|
||||
ref_res = rmsnorm(ref_emb, w, EPS) if t else ref_emb
|
||||
assert torch.equal(res, ref_res)
|
||||
assert torch.equal(attn_in, rmsnorm(ref_res, w_chain, EPS) if t else ref_res)
|
||||
|
||||
# Chained without embed_norm (use_embed_norm=False).
|
||||
res, attn_in = embed_rmsnorm(ids, table, None, EPS, chain_weight=w_chain)
|
||||
assert torch.equal(res, ref_emb)
|
||||
assert torch.equal(attn_in, rmsnorm(ref_emb, w_chain, EPS) if t else ref_emb)
|
||||
@@ -1,258 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.models.inkling.nvidia.ops.sconv as sconv
|
||||
from vllm.models.inkling.nvidia.ops import qkvr_prep
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
|
||||
|
||||
|
||||
def _make_inputs(*, is_local: bool, tokens: int = 33, tp_size: int = 4):
|
||||
torch.manual_seed(0)
|
||||
heads = 64 // tp_size
|
||||
kv_heads = (16 if is_local else 8) // tp_size
|
||||
head_dim = 128
|
||||
d_rel = 16
|
||||
rel_extent = 512 if is_local else 1024
|
||||
page_size = 16
|
||||
num_blocks = (tokens + page_size - 1) // page_size
|
||||
q_width = heads * head_dim
|
||||
kv_width = kv_heads * head_dim
|
||||
r_width = heads * d_rel
|
||||
device = "cuda"
|
||||
|
||||
qkvr = torch.randn(
|
||||
tokens,
|
||||
q_width + 2 * kv_width + r_width,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16)
|
||||
v_weight = torch.randn_like(k_weight)
|
||||
q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16)
|
||||
k_norm_weight = torch.randn_like(q_norm_weight)
|
||||
rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16)
|
||||
conv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
kv_heads,
|
||||
page_size,
|
||||
2 * head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
key_cache = torch.empty(
|
||||
num_blocks,
|
||||
page_size,
|
||||
kv_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
value_cache = torch.empty_like(key_cache)
|
||||
positions = torch.arange(tokens, device=device, dtype=torch.int64)
|
||||
block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None]
|
||||
seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32)
|
||||
slots = torch.arange(tokens, device=device, dtype=torch.int64)
|
||||
query_start = torch.zeros(tokens, device=device, dtype=torch.int32)
|
||||
log_scaling_n_floor = None if is_local else 128000
|
||||
log_scaling = None
|
||||
if log_scaling_n_floor is not None:
|
||||
log_scaling = torch.linspace(
|
||||
1.0,
|
||||
1.1,
|
||||
tokens,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return (
|
||||
qkvr,
|
||||
k_weight,
|
||||
v_weight,
|
||||
q_norm_weight,
|
||||
k_norm_weight,
|
||||
rel_proj,
|
||||
1e-6,
|
||||
heads,
|
||||
kv_heads,
|
||||
head_dim,
|
||||
d_rel,
|
||||
conv_cache,
|
||||
key_cache,
|
||||
value_cache,
|
||||
positions,
|
||||
block_table,
|
||||
seq_idx,
|
||||
slots,
|
||||
query_start,
|
||||
slots,
|
||||
0,
|
||||
head_dim,
|
||||
page_size,
|
||||
log_scaling,
|
||||
)
|
||||
|
||||
|
||||
def _reference(args):
|
||||
qkvr = args[0]
|
||||
heads, kv_heads, head_dim, d_rel = args[7:11]
|
||||
q_width = heads * head_dim
|
||||
kv_width = kv_heads * head_dim
|
||||
q, k, v, r = qkvr.split((q_width, kv_width, kv_width, heads * d_rel), dim=1)
|
||||
conv_cache = args[11].clone()
|
||||
k = sconv.fused_sconv(
|
||||
k.contiguous(),
|
||||
args[1],
|
||||
conv_cache,
|
||||
args[14],
|
||||
args[15],
|
||||
args[16],
|
||||
args[17],
|
||||
args[18],
|
||||
args[20],
|
||||
head_dim,
|
||||
args[22],
|
||||
)
|
||||
v = sconv.fused_sconv(
|
||||
v.contiguous(),
|
||||
args[2],
|
||||
conv_cache,
|
||||
args[14],
|
||||
args[15],
|
||||
args[16],
|
||||
args[17],
|
||||
args[18],
|
||||
args[21],
|
||||
head_dim,
|
||||
args[22],
|
||||
)
|
||||
|
||||
def rms_norm(x, weight):
|
||||
x = x.reshape(-1, head_dim).float()
|
||||
rstd = torch.rsqrt(x.square().mean(1, keepdim=True) + args[6])
|
||||
return (x * rstd * weight.float()).to(qkvr.dtype)
|
||||
|
||||
q = rms_norm(q, args[3]).view(qkvr.shape[0], heads, head_dim)
|
||||
k = rms_norm(k, args[4]).view(qkvr.shape[0], kv_heads, head_dim)
|
||||
v = v.view(qkvr.shape[0], kv_heads, head_dim)
|
||||
rel = torch.mm(r.reshape(-1, d_rel), args[5]).view(qkvr.shape[0], heads, -1)
|
||||
if args[23] is not None:
|
||||
q = (q.float() * args[23][:, None, None]).to(q.dtype)
|
||||
rel = (rel.float() * args[23][:, None, None]).to(rel.dtype)
|
||||
|
||||
key_cache = args[12].clone()
|
||||
value_cache = args[13].clone()
|
||||
slots = args[19]
|
||||
valid = slots >= 0
|
||||
key_cache.view(-1, kv_heads, head_dim)[slots[valid]] = k[valid]
|
||||
value_cache.view(-1, kv_heads, head_dim)[slots[valid]] = v[valid]
|
||||
return q.flatten(1), rel, key_cache, value_cache
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_cap is None,
|
||||
reason="Inkling QKVR prep kernels require CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("is_local", "tokens", "tp_size"),
|
||||
[
|
||||
(True, 9, 4),
|
||||
(False, 33, 4),
|
||||
(True, 33, 8),
|
||||
(True, 128, 4),
|
||||
(False, 128, 8),
|
||||
(True, 512, 4),
|
||||
(False, 640, 8),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_qkvr_prep_matches_reference(is_local, tokens, tp_size):
|
||||
args = _make_inputs(is_local=is_local, tokens=tokens, tp_size=tp_size)
|
||||
ref_q, ref_rel, ref_key, ref_value = _reference(args)
|
||||
|
||||
q, rel = qkvr_prep.fused_qkvr_prep(*args)
|
||||
|
||||
torch.testing.assert_close(q, ref_q, rtol=0.01, atol=0.02)
|
||||
torch.testing.assert_close(rel, ref_rel, rtol=0.02, atol=0.125)
|
||||
torch.testing.assert_close(args[12], ref_key, rtol=0.01, atol=0.01)
|
||||
torch.testing.assert_close(args[13], ref_value, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_cap is None,
|
||||
reason="Inkling QKVR prep kernels require CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize("tokens", [9, 128])
|
||||
@torch.inference_mode()
|
||||
def test_qkvr_log_scaling_preserves_bf16_norm_output(tokens):
|
||||
args = list(_make_inputs(is_local=False, tokens=tokens, tp_size=8))
|
||||
tau = args[23]
|
||||
assert tau is not None
|
||||
|
||||
scaled_q, scaled_rel = qkvr_prep.fused_qkvr_prep(*args)
|
||||
args[23] = None
|
||||
unscaled_q, unscaled_rel = qkvr_prep.fused_qkvr_prep(*args)
|
||||
|
||||
expected_q = (unscaled_q.float() * tau[:, None]).to(unscaled_q.dtype)
|
||||
expected_rel = (unscaled_rel.float() * tau[:, None, None]).to(unscaled_rel.dtype)
|
||||
torch.testing.assert_close(scaled_q, expected_q, rtol=0, atol=0)
|
||||
torch.testing.assert_close(scaled_rel, expected_rel, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_cap is None,
|
||||
reason="Inkling QKVR prep kernels require CUDA",
|
||||
)
|
||||
@pytest.mark.parametrize("tokens", [9, 128])
|
||||
@torch.inference_mode()
|
||||
def test_qkvr_prep_negative_slots_skip_cache_writes(tokens):
|
||||
args = list(_make_inputs(is_local=True, tokens=tokens))
|
||||
args[17].fill_(-1)
|
||||
args[19].fill_(-1)
|
||||
conv_cache = args[11].clone()
|
||||
key_cache = args[12].clone()
|
||||
value_cache = args[13].clone()
|
||||
|
||||
qkvr_prep.fused_qkvr_prep(*args)
|
||||
|
||||
torch.testing.assert_close(args[11], conv_cache, rtol=0, atol=0)
|
||||
torch.testing.assert_close(args[12], key_cache, rtol=0, atol=0)
|
||||
torch.testing.assert_close(args[13], value_cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@torch.inference_mode()
|
||||
def test_fused_sconv_negative_slots_skip_cache_writes():
|
||||
args = list(_make_inputs(is_local=True, tokens=9))
|
||||
args[17].fill_(-1)
|
||||
cache = args[11].clone()
|
||||
|
||||
sconv.fused_sconv(
|
||||
args[0][:, 16 * 128 : 16 * 128 + 4 * 128].contiguous(),
|
||||
args[1],
|
||||
args[11],
|
||||
args[14],
|
||||
args[15],
|
||||
args[16],
|
||||
args[17],
|
||||
args[18],
|
||||
0,
|
||||
128,
|
||||
args[22],
|
||||
)
|
||||
|
||||
torch.testing.assert_close(args[11], cache, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rel_extent", "last_latency_rows", "first_throughput_rows"),
|
||||
[(512, 8191, 8192), (1024, 2047, 2048)],
|
||||
)
|
||||
def test_rel_projection_schedule_crossover(
|
||||
rel_extent, last_latency_rows, first_throughput_rows
|
||||
):
|
||||
assert not qkvr_prep.use_rel_proj_throughput(last_latency_rows, rel_extent)
|
||||
assert qkvr_prep.use_rel_proj_throughput(first_throughput_rows, rel_extent)
|
||||
@@ -1,119 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Parity test for the fused sconv seq-metadata kernel.
|
||||
|
||||
``sconv_seq_metadata`` fills the per-token ``seq_idx`` (owning request) and
|
||||
``query_start`` (first x-row of that request) buffers in a single launch. Actual
|
||||
tokens must match the searchsorted-based reference, while CUDA graph padding
|
||||
must be initialized to safe zero values.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.inkling.nvidia.ops.sconv import sconv_seq_metadata
|
||||
from vllm.models.inkling.nvidia.sconv_swa_attn import InklingSconvMetadataBuilder
|
||||
|
||||
CASES = [
|
||||
# (query_lens, extra_pad_tokens)
|
||||
([1], 0), # bsz1 decode
|
||||
([1] * 8, 0), # uniform decode
|
||||
([1] * 8, 3), # uniform decode, padded tokens past the last request
|
||||
([2] * 4, 0), # uniform spec-decode
|
||||
([2048], 0), # single prefill
|
||||
([517, 1, 1, 33, 1, 128], 0), # mixed prefill/decode
|
||||
([517, 1, 1, 33, 1, 128], 5), # mixed, padded
|
||||
([1] * 500, 0), # many requests (deep binary search)
|
||||
]
|
||||
|
||||
|
||||
def _ref(query_start_loc: torch.Tensor, num_reqs: int, num_tokens: int):
|
||||
cu_seqlens = query_start_loc[: num_reqs + 1].to(torch.int64)
|
||||
token_idx = torch.arange(num_tokens, device=cu_seqlens.device, dtype=torch.int64)
|
||||
seq_idx = (torch.searchsorted(cu_seqlens, token_idx, right=True) - 1).clamp(
|
||||
max=num_reqs - 1
|
||||
)
|
||||
return seq_idx.to(torch.int32), cu_seqlens[seq_idx].to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("query_lens,extra_pad", CASES)
|
||||
def test_sconv_seq_metadata_matches_searchsorted(query_lens, extra_pad):
|
||||
device = "cuda"
|
||||
num_reqs = len(query_lens)
|
||||
query_start_loc = torch.tensor(
|
||||
[0] + list(torch.tensor(query_lens).cumsum(0)), dtype=torch.int32
|
||||
).to(device)
|
||||
num_actual_tokens = int(query_start_loc[-1])
|
||||
num_padded_tokens = num_actual_tokens + extra_pad
|
||||
|
||||
ref_seq, ref_qs = _ref(query_start_loc, num_reqs, num_actual_tokens)
|
||||
|
||||
seq_idx = torch.full((num_padded_tokens,), -1, dtype=torch.int32, device=device)
|
||||
query_start = torch.full_like(seq_idx, -1)
|
||||
sconv_seq_metadata(
|
||||
query_start_loc,
|
||||
num_reqs,
|
||||
num_actual_tokens,
|
||||
seq_idx,
|
||||
query_start,
|
||||
num_padded_tokens,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(seq_idx[:num_actual_tokens], ref_seq, rtol=0, atol=0)
|
||||
torch.testing.assert_close(query_start[:num_actual_tokens], ref_qs, rtol=0, atol=0)
|
||||
assert torch.count_nonzero(seq_idx[num_actual_tokens:]) == 0
|
||||
assert torch.count_nonzero(query_start[num_actual_tokens:]) == 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sconv_metadata_reuses_padded_static_buffers():
|
||||
device = torch.device("cuda")
|
||||
builder = object.__new__(InklingSconvMetadataBuilder)
|
||||
builder.seq_idx_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||
builder.query_start_buffer = torch.empty(8, dtype=torch.int32, device=device)
|
||||
|
||||
class CommonMetadata:
|
||||
num_reqs = 1
|
||||
num_actual_tokens = 8
|
||||
query_start_loc = torch.tensor([0, 3], dtype=torch.int32, device=device)
|
||||
query_start_loc_cpu = torch.tensor([0, 3], dtype=torch.int32)
|
||||
block_table_tensor = torch.zeros((1, 1), dtype=torch.int32, device=device)
|
||||
slot_mapping = torch.tensor(
|
||||
[0, 1, 2, -1, -1, -1, -1, -1],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
|
||||
common = CommonMetadata()
|
||||
first = builder.build(0, common)
|
||||
pointers = tuple(
|
||||
tensor.data_ptr()
|
||||
for tensor in (
|
||||
first.block_table,
|
||||
first.slot_mapping,
|
||||
first.seq_idx,
|
||||
first.query_start,
|
||||
)
|
||||
)
|
||||
|
||||
common.query_start_loc[1] = 5
|
||||
common.query_start_loc_cpu[1] = 5
|
||||
common.slot_mapping[:5] = torch.arange(5, dtype=torch.int64, device=device)
|
||||
second = builder.build(0, common)
|
||||
|
||||
assert second.slot_mapping.shape == (8,)
|
||||
assert second.seq_idx.shape == (8,)
|
||||
assert second.query_start.shape == (8,)
|
||||
assert pointers == tuple(
|
||||
tensor.data_ptr()
|
||||
for tensor in (
|
||||
second.block_table,
|
||||
second.slot_mapping,
|
||||
second.seq_idx,
|
||||
second.query_start,
|
||||
)
|
||||
)
|
||||
assert torch.count_nonzero(second.seq_idx[5:]) == 0
|
||||
assert torch.count_nonzero(second.query_start[5:]) == 0
|
||||
assert torch.all(second.slot_mapping[5:] == -1)
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AMD AttnRes requires ROCm",
|
||||
)
|
||||
|
||||
|
||||
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
|
||||
storage = torch.randn(
|
||||
*shape[:-1],
|
||||
shape[-1] + padding,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return storage[..., : shape[-1]]
|
||||
|
||||
|
||||
def _reference(
|
||||
prefix: torch.Tensor,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
num_blocks: int,
|
||||
eps: float,
|
||||
) -> torch.Tensor:
|
||||
hidden_size = prefix.shape[-1]
|
||||
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
|
||||
keys = F.rms_norm(values, (hidden_size,), norm_weight, eps)
|
||||
probs = (keys @ qk_weight).softmax(dim=-1)
|
||||
return torch.matmul(probs.unsqueeze(1), values).squeeze(1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"block_capacity",
|
||||
"hidden_size",
|
||||
"row_padding",
|
||||
),
|
||||
[
|
||||
pytest.param(0, 3, 5, 128, 0, id="empty"),
|
||||
pytest.param(1, 1, 2, 128, 0, id="decode-single"),
|
||||
pytest.param(17, 4, 6, 1024, 7, id="decode-padded"),
|
||||
pytest.param(320, 8, 10, 7168, 0, id="prefill-full"),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_matches_reference(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
block_capacity: int,
|
||||
hidden_size: int,
|
||||
row_padding: int,
|
||||
) -> None:
|
||||
eps = 1e-5
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=row_padding)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens,
|
||||
block_capacity,
|
||||
hidden_size,
|
||||
padding=row_padding,
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
|
||||
)
|
||||
expected = _reference(
|
||||
prefix,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
num_blocks,
|
||||
eps,
|
||||
)
|
||||
original_prefix = prefix.clone()
|
||||
original_blocks = blocks.clone()
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
None,
|
||||
num_blocks,
|
||||
-1,
|
||||
eps,
|
||||
0.0,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, original_prefix, atol=0, rtol=0)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.shape == prefix.shape
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"hidden_size",
|
||||
"has_delta",
|
||||
"write_block",
|
||||
"apply_output_norm",
|
||||
),
|
||||
[
|
||||
pytest.param(1, 0, 128, False, True, True, id="empty-write-norm"),
|
||||
pytest.param(7, 1, 1024, True, False, True, id="single-add-norm"),
|
||||
pytest.param(17, 5, 7168, True, True, True, id="padded-write-add"),
|
||||
pytest.param(3, 8, 7168, True, False, True, id="full-add-norm"),
|
||||
pytest.param(320, 4, 7168, True, False, False, id="prefill-add"),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_fused_contract(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
hidden_size: int,
|
||||
has_delta: bool,
|
||||
write_block: bool,
|
||||
apply_output_norm: bool,
|
||||
) -> None:
|
||||
torch.manual_seed(42)
|
||||
eps = 1e-5
|
||||
output_eps = 2e-5
|
||||
block_capacity = 9
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
|
||||
delta = (
|
||||
_randn_with_row_padding(num_tokens, hidden_size, padding=11)
|
||||
if has_delta
|
||||
else None
|
||||
)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens, block_capacity, hidden_size, padding=13
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
hidden_size, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) / hidden_size**0.5
|
||||
)
|
||||
output_norm_weight = (
|
||||
1 + 0.1 * torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16)
|
||||
if apply_output_norm
|
||||
else None
|
||||
)
|
||||
expected_prefix = prefix.clone()
|
||||
if delta is not None:
|
||||
expected_prefix = expected_prefix + delta
|
||||
values = torch.cat(
|
||||
(blocks[:, :num_blocks].clone(), expected_prefix.unsqueeze(1)), dim=1
|
||||
)
|
||||
keys = F.rms_norm(values.float(), (hidden_size,), norm_weight.float(), eps)
|
||||
probs = (keys @ qk_weight.float()).softmax(dim=-1)
|
||||
expected = torch.matmul(probs.unsqueeze(1), values.float()).squeeze(1)
|
||||
if output_norm_weight is not None:
|
||||
expected = F.rms_norm(
|
||||
expected, (hidden_size,), output_norm_weight.float(), output_eps
|
||||
)
|
||||
expected = expected.to(prefix.dtype)
|
||||
original_blocks = blocks.clone()
|
||||
block_write_idx = num_blocks if write_block else -1
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
eps,
|
||||
output_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
|
||||
if write_block:
|
||||
original_blocks[:, block_write_idx].copy_(expected_prefix)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,num_blocks,has_delta,write_block",
|
||||
[
|
||||
(0, 0, False, False),
|
||||
(1, 0, True, True),
|
||||
(17, 4, True, True),
|
||||
(320, 8, False, False),
|
||||
],
|
||||
)
|
||||
def test_amd_attn_res_fp8_preserves_prefix_and_quantized_output(
|
||||
num_tokens,
|
||||
num_blocks,
|
||||
has_delta,
|
||||
write_block,
|
||||
default_vllm_config,
|
||||
):
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
|
||||
torch.manual_seed(41)
|
||||
hidden_size = 7168
|
||||
prefix = _randn_with_row_padding(num_tokens, hidden_size, padding=7)
|
||||
delta = _randn_with_row_padding(num_tokens, hidden_size) if has_delta else None
|
||||
blocks = _randn_with_row_padding(num_tokens, 9, hidden_size, padding=13)
|
||||
norm = 1 + torch.randn(hidden_size, device="cuda", dtype=torch.bfloat16) * 0.1
|
||||
score = torch.randn_like(norm) / hidden_size**0.5
|
||||
out_norm = 1 + torch.randn_like(norm) * 0.1
|
||||
ref_prefix, ref_blocks = prefix.clone(), blocks.clone()
|
||||
kwargs = dict(
|
||||
num_blocks=num_blocks,
|
||||
block_write_idx=num_blocks if write_block else -1,
|
||||
eps=1e-5,
|
||||
output_norm_eps=1e-5,
|
||||
)
|
||||
reference = attn_res(ref_prefix, delta, ref_blocks, norm, score, out_norm, **kwargs)
|
||||
output, scale = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm,
|
||||
score,
|
||||
out_norm,
|
||||
quant_dtype=current_platform.fp8_dtype(),
|
||||
**kwargs,
|
||||
)
|
||||
assert output.dtype == current_platform.fp8_dtype()
|
||||
assert scale.shape == (num_tokens, 1)
|
||||
if num_tokens:
|
||||
# Keep reference rounding independent of Inductor's fused quantization.
|
||||
quant = QuantFP8(
|
||||
static=False, group_shape=GroupShape.PER_TOKEN, compile_native=False
|
||||
)
|
||||
ref_output, ref_scale = quant(reference)
|
||||
torch.testing.assert_close(scale, ref_scale, atol=1e-7, rtol=1e-6)
|
||||
|
||||
# Floating-point fusion can move values at FP8 rounding midpoints.
|
||||
# Bound each change to the adjacent code and bound how often it occurs.
|
||||
# Codes are sign-magnitude, so order them by value to keep a step
|
||||
# across zero adjacent.
|
||||
def ordinal(x: torch.Tensor) -> torch.Tensor:
|
||||
code = x.view(torch.uint8).to(torch.int16)
|
||||
return torch.where(code >= 0x80, 0x80 - code, code)
|
||||
|
||||
code_delta = (ordinal(output) - ordinal(ref_output)).abs()
|
||||
assert code_delta.max().item() <= 1
|
||||
assert (code_delta != 0).float().mean().item() < 1e-4
|
||||
torch.testing.assert_close(prefix, ref_prefix, atol=0, rtol=0)
|
||||
torch.testing.assert_close(blocks, ref_blocks, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"matching,requires_unquantized", [(True, False), (False, False), (True, True)]
|
||||
)
|
||||
def test_mla_input_quant_requires_all_consumers(matching, requires_unquantized):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
|
||||
|
||||
attention = object.__new__(KimiK3MultiHeadLatentAttentionWrapper)
|
||||
nn.Module.__init__(attention)
|
||||
attention.indexer = None
|
||||
attention.q_lora_rank = 1536
|
||||
attention.fused_qkv_a_proj = nn.Identity()
|
||||
attention.g_proj = nn.Identity()
|
||||
kernel = SimpleNamespace(input_quant_key=lambda: kFp8DynamicTokenSym)
|
||||
expose_input_quant_key(attention.fused_qkv_a_proj, kernel)
|
||||
if matching:
|
||||
expose_input_quant_key(attention.g_proj, kernel)
|
||||
attention.g_proj.requires_unquantized_input = requires_unquantized
|
||||
actual = attention.get_input_quant_key()
|
||||
assert actual == (
|
||||
kFp8DynamicTokenSym if matching and not requires_unquantized else None
|
||||
)
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm Kimi-K3 KDA metadata under adaptive verification.
|
||||
|
||||
Adaptive verification splits the draft budget evenly on the host, then trims
|
||||
each verify request on device. The KDA spec path must therefore plan from the
|
||||
device offsets, never from host lengths that merely look uniform.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import (
|
||||
KimiK3ROCmKDABackend,
|
||||
KimiK3ROCmKDAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
BLOCK_SIZE = 16
|
||||
DEVICE = torch.device("cpu")
|
||||
NUM_SPEC = 3
|
||||
|
||||
|
||||
def _config(adaptive: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B",
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram",
|
||||
num_speculative_tokens=NUM_SPEC,
|
||||
enable_adaptive_verification=adaptive,
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = "none"
|
||||
return vllm_config
|
||||
|
||||
|
||||
def _spec() -> MambaSpec:
|
||||
return MambaSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
num_speculative_blocks=NUM_SPEC,
|
||||
)
|
||||
|
||||
|
||||
def _builder(adaptive: bool) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
return KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=_spec(),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=_config(adaptive),
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _trimmed_verify_batch(builder: KimiK3ROCmKDAMetadataBuilder):
|
||||
"""Host sees [3, 3]; the device was trimmed to [4, 2] (same total)."""
|
||||
batch = BatchSpec(seq_lens=[64, 64], query_lens=[4, 2])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, DEVICE, arange_block_indices=True
|
||||
)
|
||||
common = common.replace(
|
||||
query_start_loc_cpu=torch.tensor([0, 3, 6], dtype=torch.int32)
|
||||
)
|
||||
accepted = torch.ones(2, dtype=torch.int32, device=DEVICE)
|
||||
num_decode_draft_tokens_cpu = torch.tensor([NUM_SPEC, NUM_SPEC], dtype=torch.int32)
|
||||
return builder.build(0, common, accepted, num_decode_draft_tokens_cpu)
|
||||
|
||||
|
||||
def test_adaptive_verification_plans_from_device_offsets():
|
||||
"""Equal host lengths would select the fixed-length recurrent kernel,
|
||||
which places sequence i at i * L and ignores the device cu_seqlens."""
|
||||
assert KimiK3ROCmKDABackend.supports_device_cpu_query_lens_mismatch()
|
||||
bound = KimiK3ROCmKDAMetadataBuilder.get_varlen_cudagraph_max_query_len
|
||||
assert bound(_config(adaptive=True), _spec()) == NUM_SPEC + 1
|
||||
assert bound(_config(adaptive=False), _spec()) is None
|
||||
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=True))
|
||||
assert meta.num_spec_decodes == 2
|
||||
assert meta.uniform_spec_sequence_length is None
|
||||
assert torch.equal(
|
||||
meta.spec_query_start_loc.cpu(), torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
)
|
||||
# Without adaptive verification the host lengths are exact and kept.
|
||||
meta = _trimmed_verify_batch(_builder(adaptive=False))
|
||||
assert meta.uniform_spec_sequence_length == 3
|
||||
@@ -1,527 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm KDA prefill checkpoint: where it is taken and what it stores.
|
||||
|
||||
Two things have to line up for a checkpoint to be safe: the metadata must name
|
||||
a position the chunk kernel will actually write, and the conv window saved
|
||||
there must be the one a resumed prefill would have left.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDAMetadataBuilder
|
||||
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
KDA_CHECKPOINT_ALIGNMENT,
|
||||
is_fused_kda_chunk_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_rocm()
|
||||
and torch.cuda.is_available()
|
||||
and is_fused_kda_chunk_supported()
|
||||
),
|
||||
reason="The ROCm KDA prefill checkpoint is exported by the fused chunk "
|
||||
"kernel, which needs a gfx950 build that includes it",
|
||||
)
|
||||
|
||||
# A mamba block size the chunk walk lands on. The real K3 value is 768; any
|
||||
# multiple of the kernel's 64-token chunk behaves the same and keeps the
|
||||
# fixtures small.
|
||||
BLOCK_SIZE = 64
|
||||
|
||||
|
||||
class _StubSpeculativeConfig:
|
||||
method = "dspark"
|
||||
parallel_drafting = False
|
||||
|
||||
def __init__(self, num_speculative_tokens: int) -> None:
|
||||
self.num_speculative_tokens = num_speculative_tokens
|
||||
|
||||
def use_eagle_block_drop(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _builder(
|
||||
num_prefill_checkpoint_blocks: int = 1,
|
||||
mamba_cache_mode: str = "align",
|
||||
block_size: int = BLOCK_SIZE,
|
||||
prefix_match_unit: int | None = None,
|
||||
num_spec: int = 0,
|
||||
) -> KimiK3ROCmKDAMetadataBuilder:
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3.5-0.8B", block_size=block_size
|
||||
)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
|
||||
if num_spec:
|
||||
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
|
||||
builder = KimiK3ROCmKDAMetadataBuilder(
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=(
|
||||
KDA_CHECKPOINT_ALIGNMENT if num_prefill_checkpoint_blocks else None
|
||||
),
|
||||
num_speculative_blocks=num_spec,
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=torch.device("cuda"),
|
||||
)
|
||||
vllm_config.cache_config.hash_block_size = prefix_match_unit or block_size
|
||||
vllm_config.cache_config.cache_hit_alignment_tokens = (
|
||||
prefix_match_unit or block_size
|
||||
)
|
||||
return builder
|
||||
|
||||
|
||||
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
|
||||
common = create_common_attn_metadata(
|
||||
batch, block_size, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
return _builder(block_size=block_size, **kw).build(0, common)
|
||||
|
||||
|
||||
def _i32(values: list[int]) -> torch.Tensor:
|
||||
return torch.tensor(values, dtype=torch.int32, device="cuda")
|
||||
|
||||
|
||||
def test_checkpoint_metadata_targets_last_aligned_boundary() -> None:
|
||||
# Slot p holds the state after (p + 1) * block_size tokens, so a 200-token
|
||||
# prompt checkpoints at 192 into slot 2 and a 128-token one at 64 into slot
|
||||
# 0. The block table is an arange, so request 1 owns blocks 4..7.
|
||||
md = _build(BatchSpec(seq_lens=[200, 128], query_lens=[200, 128]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 64]))
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([2, 4]))
|
||||
|
||||
|
||||
def test_checkpoint_offset_is_relative_to_query_start() -> None:
|
||||
# 256 tokens computed, 144 more this step: the boundary at 384 lands 128
|
||||
# tokens into the chunk, and cdiv(400, 64) - 2 = 5 is its slot.
|
||||
md = _build(BatchSpec(seq_lens=[400, 64], query_lens=[144, 64]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([128, 0]))
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([5, NULL_BLOCK_ID]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_avoids_live_state_row() -> None:
|
||||
# Writing the row the walk itself owns would race it.
|
||||
md = _build(BatchSpec(seq_lens=[200, 130], query_lens=[200, 130]))
|
||||
|
||||
assert md.checkpoint is not None
|
||||
# `mamba_get_block_table_tensor` hands the walk block (seq_len - 1) // bs.
|
||||
live = torch.tensor([(200 - 1) // BLOCK_SIZE, (130 - 1) // BLOCK_SIZE])
|
||||
table = torch.arange(2 * 4).view(2, 4)
|
||||
live_rows = table[torch.arange(2), live].cuda()
|
||||
assert not bool((md.checkpoint.state_indices == live_rows).any())
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_chunk_without_boundary() -> None:
|
||||
# 128 computed, 40 more: the only boundary at 128 is the state this chunk
|
||||
# started from, so there is nothing new to snapshot.
|
||||
assert _build(BatchSpec(seq_lens=[168], query_lens=[40])).checkpoint is None
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_null_reservation() -> None:
|
||||
# Only the chunk that finishes a prompt gets a block reserved; every
|
||||
# earlier one leaves the shared null block, which nothing hashes and
|
||||
# writing it would corrupt every other request's scratch row.
|
||||
batch = BatchSpec(seq_lens=[200], query_lens=[200])
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
# cdiv(200, 64) - 2 = 2 is the checkpoint slot.
|
||||
common.block_table_tensor[0, 2] = NULL_BLOCK_ID
|
||||
|
||||
md = _builder().build(0, common)
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([NULL_BLOCK_ID]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_absent_without_reserved_block() -> None:
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[200], query_lens=[200]),
|
||||
num_prefill_checkpoint_blocks=0,
|
||||
)
|
||||
assert getattr(md, "checkpoint", None) is None
|
||||
|
||||
|
||||
def test_spec_checkpoint_metadata_gathers_non_spec_rows_by_index() -> None:
|
||||
# gdn_attn selects the non-spec rows by mask and reclassifies their
|
||||
# one-token decodes as prefills, so the chunk kernel is handed rows 1, 3
|
||||
# and 4 here while a num_decodes: slice would hand it rows 0, 1 and 2.
|
||||
num_spec = 2
|
||||
batch = BatchSpec(
|
||||
seq_lens=[300, 200, 400, 512, 128],
|
||||
query_lens=[num_spec + 1, 200, num_spec + 1, 1, 128],
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
|
||||
)
|
||||
# Align mode reads 1 + num_speculative_blocks columns, so widen the table.
|
||||
rows, cols = common.block_table_tensor.shape
|
||||
width = cols + num_spec
|
||||
common.block_table_tensor = torch.arange(
|
||||
rows * width, dtype=torch.int32, device="cuda"
|
||||
).view(rows, width)
|
||||
|
||||
md = _builder(num_spec=num_spec).build(
|
||||
0,
|
||||
common,
|
||||
num_accepted_tokens=torch.ones(rows, dtype=torch.int32, device="cuda"),
|
||||
num_decode_draft_tokens_cpu=torch.tensor(
|
||||
[num_spec, -1, num_spec, -1, -1], dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
assert md.spec_sequence_masks is not None
|
||||
assert md.num_decodes == 0 and md.num_prefills == 3
|
||||
assert md.checkpoint is not None
|
||||
# Row 3 resumes mid-block, so its chunk holds no reusable boundary.
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 0, 64]))
|
||||
# Request r owns blocks r*width.., so slot cdiv(200, 64) - 2 = 2 for
|
||||
# request 1 and cdiv(128, 64) - 2 = 0 for request 4.
|
||||
torch.testing.assert_close(
|
||||
md.checkpoint.state_indices, _i32([width + 2, NULL_BLOCK_ID, 4 * width])
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_metadata_absent_outside_align_mode() -> None:
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[200], query_lens=[200]),
|
||||
mamba_cache_mode="none",
|
||||
)
|
||||
assert getattr(md, "checkpoint", None) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conv half
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DIM = 384
|
||||
WIDTH = 4
|
||||
STATE_LEN = WIDTH - 1
|
||||
|
||||
|
||||
def store_conv_checkpoints(x, conv_state, cu, offsets, rows, state_len) -> None:
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter(state_len=state_len).export(
|
||||
MambaPrefillCheckpointMetadata(offsets, rows),
|
||||
raw_qkv=x,
|
||||
conv_state=conv_state,
|
||||
cu_seqlens=cu,
|
||||
)
|
||||
|
||||
|
||||
def _conv_fixture(
|
||||
seqlens: list[int], slots: int = 16, seed: int = 0, num_spec: int = 0
|
||||
):
|
||||
# kda_state_shape widens the conv row to conv_kernel_size - 1 + num_spec,
|
||||
# but the window every reader starts from stays the leading STATE_LEN.
|
||||
torch.manual_seed(seed)
|
||||
total = sum(seqlens)
|
||||
x = torch.randn(total, DIM, device="cuda", dtype=torch.bfloat16)
|
||||
cu = torch.tensor(
|
||||
[0] + torch.tensor(seqlens).cumsum(0).tolist(),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
conv_state = torch.full(
|
||||
(slots, DIM, STATE_LEN + num_spec),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return x, cu, conv_state
|
||||
|
||||
|
||||
def test_conv_checkpoint_saves_window_before_offset() -> None:
|
||||
seqlens = [200, 130]
|
||||
x, cu, conv_state = _conv_fixture(seqlens)
|
||||
offsets = _i32([192, 128])
|
||||
rows = _i32([5, 9])
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, offsets, rows, STATE_LEN)
|
||||
|
||||
for seq, (offset, row) in enumerate(zip([192, 128], [5, 9])):
|
||||
end = int(cu[seq]) + offset
|
||||
expected = x[end - STATE_LEN : end].transpose(0, 1)
|
||||
torch.testing.assert_close(conv_state[row], expected)
|
||||
|
||||
|
||||
def test_conv_checkpoint_skips_opted_out_sequences() -> None:
|
||||
x, cu, conv_state = _conv_fixture([200, 200])
|
||||
before = conv_state.clone()
|
||||
|
||||
store_conv_checkpoints(
|
||||
x, conv_state, cu, _i32([0, 192]), _i32([5, NULL_BLOCK_ID]), STATE_LEN
|
||||
)
|
||||
|
||||
assert torch.equal(conv_state.isnan(), before.isnan())
|
||||
|
||||
|
||||
def test_exporter_stores_conv_window_through_shared_interface() -> None:
|
||||
x, cu, conv_state = _conv_fixture([200, 200])
|
||||
exporter = KimiK3ROCmKDAPrefillCheckpointExporter(state_len=STATE_LEN)
|
||||
assert isinstance(exporter, MambaPrefillCheckpointExporter)
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
_i32([192, 0]), _i32([5, NULL_BLOCK_ID])
|
||||
)
|
||||
|
||||
exporter.export(checkpoint, raw_qkv=x, conv_state=conv_state, cu_seqlens=cu)
|
||||
|
||||
torch.testing.assert_close(conv_state[5], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
|
||||
untouched[5] = False
|
||||
assert bool(conv_state[untouched].isnan().all())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_spec", [0, 2, 8])
|
||||
def test_conv_checkpoint_matches_prefill_truncated_at_offset(num_spec: int) -> None:
|
||||
# causal_conv1d_fn's own state write is the reference: a prefill that
|
||||
# stopped at the offset leaves exactly the row a later prefix-cache hit
|
||||
# reads back. Speculative decoding widens the row without moving that
|
||||
# window, so a checkpoint sized off the row would sit num_spec tokens in
|
||||
# the past.
|
||||
seqlens = [200, 130]
|
||||
offsets = [192, 64]
|
||||
x, cu, conv_state = _conv_fixture(seqlens, seed=3, num_spec=num_spec)
|
||||
weight = torch.randn(DIM, WIDTH, device="cuda", dtype=torch.bfloat16) * 0.3
|
||||
bias = torch.randn(DIM, device="cuda", dtype=torch.bfloat16)
|
||||
rows = [5, 9]
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32(offsets), _i32(rows), STATE_LEN)
|
||||
|
||||
# A prefill of just the prefix, into its own cache. Row 0 is the null block
|
||||
# the conv kernel skips by design, so the reference rows start at 1.
|
||||
prefix = torch.cat(
|
||||
[x[int(cu[n]) : int(cu[n]) + off] for n, off in enumerate(offsets)]
|
||||
)
|
||||
prefix_cu = torch.tensor(
|
||||
[0] + torch.tensor(offsets).cumsum(0).tolist(),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
reference = torch.zeros_like(conv_state)
|
||||
causal_conv1d_fn(
|
||||
prefix.transpose(0, 1),
|
||||
weight,
|
||||
bias,
|
||||
activation="silu",
|
||||
conv_states=reference,
|
||||
has_initial_state=torch.zeros(2, dtype=torch.bool, device="cuda"),
|
||||
cache_indices=_i32([1, 2]),
|
||||
query_start_loc=prefix_cu,
|
||||
)
|
||||
|
||||
# causal_conv1d_fn writes only the leading STATE_LEN columns.
|
||||
window = slice(None), slice(None, STATE_LEN)
|
||||
torch.testing.assert_close(conv_state[rows[0]][window], reference[1][window])
|
||||
torch.testing.assert_close(conv_state[rows[1]][window], reference[2][window])
|
||||
|
||||
|
||||
def test_conv_checkpoint_leaves_spec_tail_of_row_alone() -> None:
|
||||
# The draft-token columns belong to causal_conv1d_update. Filling the whole
|
||||
# row would hand every reader, which indexes from column 0, a window
|
||||
# shifted num_spec tokens into the past.
|
||||
num_spec = 4
|
||||
x, cu, conv_state = _conv_fixture([200], num_spec=num_spec, seed=11)
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([5]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(
|
||||
conv_state[5, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
|
||||
)
|
||||
assert bool(conv_state[5, :, STATE_LEN:].isnan().all()), (
|
||||
"the checkpoint wrote into the speculative-decode tail of the row"
|
||||
)
|
||||
|
||||
|
||||
def test_conv_checkpoint_honours_transposed_cache_view() -> None:
|
||||
# The SD conv layout reaches the layer transposed, not contiguous.
|
||||
x, cu, _ = _conv_fixture([200])
|
||||
raw = torch.full(
|
||||
(16, STATE_LEN, DIM), float("nan"), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
conv_state = raw.transpose(-1, -2)
|
||||
assert not conv_state.is_contiguous()
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([7]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(conv_state[7], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
|
||||
untouched[7] = False
|
||||
assert bool(raw[untouched].isnan().all())
|
||||
|
||||
|
||||
def test_checkpoint_metadata_indexes_prefill_rows_after_decodes() -> None:
|
||||
# The chunk kernel receives the prefill tail alone, so the metadata has to
|
||||
# be indexed the same way or it snapshots the wrong request's boundary.
|
||||
md = _build(BatchSpec(seq_lens=[64, 200], query_lens=[1, 200]))
|
||||
|
||||
assert md.num_decodes == 1 and md.num_prefills == 1
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192]))
|
||||
# Request 1 owns blocks 4..7, so its slot-2 state block is 6.
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([6]))
|
||||
|
||||
|
||||
def test_conv_checkpoint_addresses_row_past_int32_limit() -> None:
|
||||
# state_stride_0 is the whole mamba page, padded up to the attention page,
|
||||
# so row * stride passes 2**31 a few thousand blocks into a large cache and
|
||||
# an int32 wrap stores outside the tensor. The margin on the plain prefill
|
||||
# path is 1.6x, which is luck: a bigger cache, a higher utilization or the
|
||||
# wider row speculative decoding allocates all close it.
|
||||
stride_row = DIM * (STATE_LEN + 8)
|
||||
row = 2**31 // stride_row + 1
|
||||
need = (row + 1) * stride_row * 2
|
||||
free, _ = torch.accelerator.get_memory_info()
|
||||
if free < need + (4 << 30):
|
||||
pytest.skip(f"needs {need >> 30} GiB of free VRAM to reach the wrap")
|
||||
|
||||
x, cu, _ = _conv_fixture([200])
|
||||
conv_state = torch.zeros(
|
||||
row + 1, DIM, STATE_LEN + 8, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
assert conv_state.stride(0) * row > 2**31, "fixture does not reach the wrap"
|
||||
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([row]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(
|
||||
conv_state[row, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
|
||||
)
|
||||
assert not bool(conv_state[:row].any()), "the store wrote outside its row"
|
||||
|
||||
|
||||
def test_conv_checkpoint_reads_row_strided_projection() -> None:
|
||||
# mixed_qkv is a band of the fused QKVGFAB projection, so its row stride is
|
||||
# the whole projection width, not DIM.
|
||||
torch.manual_seed(5)
|
||||
total, pad = 200, 177
|
||||
projection = torch.randn(total, DIM + pad, device="cuda", dtype=torch.bfloat16)
|
||||
x = projection[:, :DIM]
|
||||
assert x.stride(0) == DIM + pad
|
||||
|
||||
cu = _i32([0, total])
|
||||
conv_state = torch.full(
|
||||
(16, DIM, STATE_LEN), float("nan"), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([4]), STATE_LEN)
|
||||
|
||||
torch.testing.assert_close(conv_state[4], x[192 - STATE_LEN : 192].transpose(0, 1))
|
||||
|
||||
|
||||
def test_finer_prefix_match_unit_moves_checkpoint_to_hash_grid() -> None:
|
||||
# A partial prefix hit matches at prefix_match_unit, so a state captured
|
||||
# only at block boundaries is too coarse to resume from. Block 256 with
|
||||
# unit 128 moves a 700-token prompt's checkpoint from 512 to 640.
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[700], query_lens=[700]),
|
||||
block_size=256,
|
||||
prefix_match_unit=128,
|
||||
)
|
||||
|
||||
assert md.checkpoint is not None
|
||||
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([640]))
|
||||
# Slot is still block-indexed: cdiv(700, 256) - 2 = 1, request 0 owns 0..2.
|
||||
torch.testing.assert_close(md.checkpoint.state_indices, _i32([1]))
|
||||
|
||||
|
||||
def test_checkpoint_metadata_skips_unaligned_offset() -> None:
|
||||
# The scheduler declines a boundary off the published alignment and
|
||||
# reserves nothing, so the builder has to reach the same verdict or it
|
||||
# writes a block nobody hashed.
|
||||
# Block 192, unit 96: a 700-token prompt's last hash boundary is 672, which
|
||||
# is 32 tokens off the chunk grid.
|
||||
md = _build(
|
||||
BatchSpec(seq_lens=[700], query_lens=[700]),
|
||||
block_size=192,
|
||||
prefix_match_unit=96,
|
||||
)
|
||||
|
||||
assert md.checkpoint is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Opting in
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _spec_for(monkeypatch, **attrs) -> MambaSpec:
|
||||
"""Run the layer's `get_kv_cache_spec` over a stubbed base spec."""
|
||||
base = MambaSpec(block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,))
|
||||
monkeypatch.setattr(
|
||||
GatedDeltaNetAttention, "get_kv_cache_spec", lambda self, cfg: base
|
||||
)
|
||||
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
|
||||
for name, value in dict(
|
||||
use_prefill_checkpoint=True,
|
||||
kda_prefill_backend="fused",
|
||||
num_spec=0,
|
||||
use_safe_gate=True,
|
||||
).items():
|
||||
setattr(layer, name, attrs.get(name, value))
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
|
||||
vllm_config.cache_config.mamba_cache_mode = "align"
|
||||
return layer.get_kv_cache_spec(vllm_config)
|
||||
|
||||
|
||||
def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
|
||||
# The scheduler learns the kernel's granularity only from the spec, so
|
||||
# reserving a block without publishing the alignment would hash a state
|
||||
# that is never written.
|
||||
spec = _spec_for(monkeypatch)
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 1
|
||||
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attr,value",
|
||||
[
|
||||
("use_prefill_checkpoint", False),
|
||||
("kda_prefill_backend", "triton"),
|
||||
("kda_prefill_backend", "gluon"),
|
||||
("use_safe_gate", False),
|
||||
],
|
||||
)
|
||||
def test_paths_that_cannot_export_stay_opted_out(monkeypatch, attr, value) -> None:
|
||||
spec = _spec_for(monkeypatch, **{attr: value})
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 0
|
||||
assert spec.prefill_checkpoint_alignment is None
|
||||
|
||||
|
||||
def test_speculative_decoding_does_not_opt_out(monkeypatch) -> None:
|
||||
# num_spec reaches the kernels as a wider conv row and an interleaved
|
||||
# batch, both of which the layer handles.
|
||||
spec = _spec_for(monkeypatch, num_spec=8)
|
||||
|
||||
assert spec.num_prefill_checkpoint_blocks == 1
|
||||
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
|
||||
@@ -1,930 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The fused ROCm KDA chunk kernel must match the Triton chunk path.
|
||||
|
||||
The fused kernel reassociates nothing, but it keeps the chunk states in
|
||||
registers and rounds them to bfloat16 only where the Triton path does, so the
|
||||
outputs agree to bfloat16 rounding and the final states nearly exactly.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _fused_chunk_available() -> bool:
|
||||
if not (current_platform.is_rocm() and torch.cuda.is_available()):
|
||||
return False
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported
|
||||
|
||||
return is_fused_kda_chunk_supported()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _fused_chunk_available(),
|
||||
reason="The fused KDA chunk kernel needs a gfx950 build that includes it",
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
NUM_HEADS = 12
|
||||
LOWER_BOUND = -5.0
|
||||
|
||||
|
||||
def _requires_kernel() -> None:
|
||||
if not hasattr(torch.ops._C, "fused_kda_chunk"):
|
||||
pytest.skip("vLLM was built without the fused KDA chunk kernel")
|
||||
|
||||
|
||||
def _inputs(seqlens: list[int], seed: int) -> dict:
|
||||
torch.manual_seed(seed)
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
total = sum(seqlens)
|
||||
cu = torch.tensor(
|
||||
[0] + torch.tensor(seqlens).cumsum(0).tolist(), device=dev, dtype=torch.int32
|
||||
)
|
||||
shape = (1, total, NUM_HEADS, HEAD_DIM)
|
||||
return dict(
|
||||
q=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
k=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
raw_g=torch.randn(shape, device=dev, dtype=dtype),
|
||||
raw_beta=torch.randn(1, total, NUM_HEADS, device=dev, dtype=dtype),
|
||||
A_log=torch.randn(NUM_HEADS, device=dev, dtype=torch.float32) * 0.5,
|
||||
dt_bias=torch.randn(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
* 0.5,
|
||||
h0=torch.randn(
|
||||
len(seqlens), NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
|
||||
)
|
||||
* 0.1,
|
||||
cu=cu,
|
||||
)
|
||||
|
||||
|
||||
def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
|
||||
return chunk_kda_prefill(
|
||||
q=inp["q"].clone(),
|
||||
k=inp["k"].clone(),
|
||||
v=inp["v"].clone(),
|
||||
raw_g=inp["raw_g"].clone(),
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
g_bias=inp["dt_bias"],
|
||||
lower_bound=LOWER_BOUND,
|
||||
initial_state=inp["h0"].clone(),
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
backend=kw.pop("backend", "fused" if use_fused else "triton"),
|
||||
out=out,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
def _force_groups(monkeypatch, groups: int) -> None:
|
||||
"""Pin the group count, which is otherwise a function of the device."""
|
||||
import vllm.models.kimi_k3.amd.ops.kda_chunk as kda_chunk
|
||||
|
||||
monkeypatch.setattr(kda_chunk, "_chunk_groups", lambda *_: groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[
|
||||
[1], # shorter than a chunk
|
||||
[64], # exactly one chunk
|
||||
[1024],
|
||||
[4096], # many chunks, one sequence
|
||||
[513, 64, 1, 1200], # ragged, including a partial tail chunk
|
||||
[512] * 6,
|
||||
],
|
||||
)
|
||||
def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs(seqlens, seed=len(seqlens))
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[[1], [64], [1024], [4096], [513, 64, 1, 1200], [512] * 6],
|
||||
)
|
||||
def test_gluon_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
"""AITER's Gluon chunk_kda must match the Triton chunk path too."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
pytest.skip("AITER Gluon chunk_kda is not available here")
|
||||
|
||||
inp = _inputs(seqlens, seed=len(seqlens))
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_gluon, state_gluon = _run(inp, use_fused=False, backend="gluon")
|
||||
|
||||
torch.testing.assert_close(o_gluon.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_gluon, state_ref, rtol=5e-3, atol=5e-3)
|
||||
state_rel = (state_gluon - state_ref).norm() / state_ref.norm()
|
||||
assert state_rel < 5e-3, f"final state relative L2 error {state_rel:.2e}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_out_argument_receives_the_result(use_fused: bool) -> None:
|
||||
"""``out`` is a promise about where the result lands.
|
||||
|
||||
The layer passes a slice of its own output buffer so neither path has to
|
||||
copy afterwards, and it then skips the copy based on the returned tensor
|
||||
aliasing that buffer. Both must hold for the fused path and for the Triton
|
||||
fallback, which allocates its own buffer internally.
|
||||
"""
|
||||
if use_fused:
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [321, 64, 700]
|
||||
inp = _inputs(seqlens, seed=13)
|
||||
o_ref, _ = _run(inp, use_fused=use_fused)
|
||||
|
||||
# A non-zero offset into a larger buffer, as the layer produces when a
|
||||
# mixed batch puts decode tokens first.
|
||||
total = sum(seqlens)
|
||||
buf = torch.empty(
|
||||
1, total + 8, NUM_HEADS, HEAD_DIM, device="cuda", dtype=o_ref.dtype
|
||||
)
|
||||
view = buf[:, 8:]
|
||||
o_got, _ = _run(inp, use_fused=use_fused, out=view)
|
||||
|
||||
assert o_got.data_ptr() == view.data_ptr()
|
||||
torch.testing.assert_close(view.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_fused_chunk_accepts_projection_views() -> None:
|
||||
"""`beta` and `g` arrive as last-dim slices of the fused QKVGFAB projection.
|
||||
|
||||
Those views carry the projection's row stride, so a kernel that indexes them
|
||||
densely reads the wrong elements or rejects the tensor outright. Build the
|
||||
inputs the way the layer does rather than as standalone tensors.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [777, 64, 391]
|
||||
inp = _inputs(seqlens, seed=7)
|
||||
total = sum(seqlens)
|
||||
|
||||
# One wide projection, split on the last dim exactly like KimiK3DeltaAttention.
|
||||
qkv_width = 3 * NUM_HEADS * HEAD_DIM
|
||||
projection = torch.randn(
|
||||
total, qkv_width + NUM_HEADS, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
beta_view = projection.split([qkv_width, NUM_HEADS], dim=-1)[1].unsqueeze(0)
|
||||
assert not beta_view.is_contiguous()
|
||||
|
||||
inp["raw_beta"] = beta_view.contiguous()
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
|
||||
inp["raw_beta"] = beta_view
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def test_fused_chunk_survives_correlated_keys_and_real_gate_bias() -> None:
|
||||
"""The input regime the served model actually produces.
|
||||
|
||||
This is what forces the intra-chunk inverse to keep fp32 MFMA operands: at
|
||||
bf16 the Neumann doubling's intermediate powers cancel catastrophically
|
||||
here, diverging by orders of magnitude and driving the final state to inf.
|
||||
|
||||
Three properties of real traffic combine here and in none of the tests
|
||||
above, all measured from tensors captured off a live TP8 server:
|
||||
|
||||
* repetitive text gives near-duplicate L2-normalized keys, so ``k_i . k_j``
|
||||
approaches 1 and the Neumann terms stop shrinking;
|
||||
* the checkpoint's ``dt_bias`` lies in ``[-7.8, -1.4]``, which makes the
|
||||
within-chunk gate cumsum span ~300 powers of two;
|
||||
* ``beta`` saturates near 0.95.
|
||||
|
||||
``_inputs`` draws all three from ``randn``, landing in the benign regime.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
torch.manual_seed(11)
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
total = 1088
|
||||
shape = (1, total, NUM_HEADS, HEAD_DIM)
|
||||
base = torch.randn(1, 1, NUM_HEADS, HEAD_DIM, device=dev, dtype=dtype)
|
||||
|
||||
def correlated():
|
||||
return (base + torch.randn(shape, device=dev, dtype=dtype) * 0.02).contiguous()
|
||||
|
||||
inp = dict(
|
||||
q=correlated(),
|
||||
k=correlated(),
|
||||
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
raw_g=torch.randn(shape, device=dev, dtype=dtype),
|
||||
raw_beta=torch.full((1, total, NUM_HEADS), 3.0, device=dev, dtype=dtype),
|
||||
A_log=torch.rand(NUM_HEADS, device=dev, dtype=torch.float32) * 0.6 - 0.42,
|
||||
dt_bias=torch.rand(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32) * 6.4
|
||||
- 7.8,
|
||||
h0=torch.zeros(
|
||||
1, NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
|
||||
),
|
||||
cu=torch.tensor([0, total], device=dev, dtype=torch.int32),
|
||||
)
|
||||
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
# Looser than the tests above: near-identical keys plus a saturated beta
|
||||
# are past anything real text produces, and the inverse is rounded to bf16
|
||||
# before the w/u GEMMs. The tolerance still catches divergence, which in
|
||||
# this regime is orders of magnitude rather than a few times the bound.
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=5e-2, atol=5e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=5e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[
|
||||
[1], # every conv tap comes from the cache
|
||||
[2], # cache and batch both contribute
|
||||
[64],
|
||||
[1024],
|
||||
[513, 64, 1, 1200], # ragged, tail chunk shorter than the conv width
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("has_state", [False, True])
|
||||
def test_fused_conv_matches_reference_conv(seqlens: list[int], has_state: bool) -> None:
|
||||
"""The prologue's optional fused convolution must match a separate conv.
|
||||
|
||||
The layer does not use this path -- recomputing the conv once per load
|
||||
phase costs more than the ``causal_conv1d_fn`` launches it saves -- so it
|
||||
is driven through the ops directly rather than through
|
||||
``chunk_kda_prefill``, which does not expose it.
|
||||
|
||||
The reference is computed in torch rather than with ``causal_conv1d_fn``,
|
||||
which is not self-consistent for these shapes on ROCm (it leaves part of
|
||||
its output unwritten, so repeated identical calls disagree).
|
||||
|
||||
The cases that matter are the ones touching the sequence boundary, where
|
||||
the fused path takes taps from the conv cache instead of the batch:
|
||||
sequences shorter than the kernel width, and a cache that is or is not
|
||||
populated. The rolled-forward cache is checked too, since a wrong shift
|
||||
only shows up on the next step.
|
||||
"""
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
fused_kda_chunk,
|
||||
fused_kda_prologue,
|
||||
)
|
||||
|
||||
_requires_kernel()
|
||||
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
lp = NUM_HEADS * HEAD_DIM
|
||||
width, n = 4, len(seqlens)
|
||||
inp = _inputs(seqlens, seed=7 + len(seqlens) + int(has_state))
|
||||
total = int(inp["cu"][-1])
|
||||
|
||||
torch.manual_seed(11 + int(has_state))
|
||||
# The layer hands the prologue a strided view of the QKVGFAB projection.
|
||||
projection = torch.randn(total, 3 * lp + 64, device=dev, dtype=dtype) * 0.5
|
||||
mixed = projection[:, : 3 * lp]
|
||||
# conv1d is built with bias=False and fp32 weights; the fused kernel takes
|
||||
# the width-major [qkv, width, channel] mirror the decode path already
|
||||
# keeps.
|
||||
weight = torch.randn(3, width, lp, device=dev, dtype=torch.float32) * 0.3
|
||||
store = torch.zeros(n, width - 1, 3 * lp, device=dev, dtype=dtype)
|
||||
if has_state:
|
||||
store.normal_(0.0, 0.5)
|
||||
state = store.transpose(-1, -2)
|
||||
idx = torch.arange(n, device=dev, dtype=torch.int32)
|
||||
has_init = torch.full((n,), has_state, device=dev, dtype=torch.bool)
|
||||
|
||||
# Reference conv, per sequence, in fp32 then rounded like a bf16 output.
|
||||
conv_ref = torch.empty(total, 3 * lp, device=dev, dtype=dtype)
|
||||
state_ref = torch.empty_like(store)
|
||||
flat_w = weight.permute(0, 2, 1).reshape(3 * lp, width)
|
||||
for i, (b, e) in enumerate(zip(inp["cu"][:-1].tolist(), inp["cu"][1:].tolist())):
|
||||
pre = (
|
||||
store[i].float()
|
||||
if has_state
|
||||
else torch.zeros(width - 1, 3 * lp, device=dev, dtype=torch.float32)
|
||||
)
|
||||
xs = torch.cat([pre, mixed[b:e].float()], 0)
|
||||
acc = torch.zeros(e - b, 3 * lp, device=dev, dtype=torch.float32)
|
||||
for w in range(width):
|
||||
acc += xs[w : w + (e - b)] * flat_w[:, w].unsqueeze(0)
|
||||
conv_ref[b:e] = F.silu(acc).to(dtype)
|
||||
state_ref[i] = xs[-(width - 1) :].to(dtype)
|
||||
|
||||
def _bands(x, contiguous):
|
||||
out = [
|
||||
b.unsqueeze(0).unflatten(-1, (NUM_HEADS, HEAD_DIM))
|
||||
for b in x.split(lp, dim=-1)
|
||||
]
|
||||
return [b.contiguous() for b in out] if contiguous else out
|
||||
|
||||
# Compare through both kernels rather than the prologue workspaces: `aqk`
|
||||
# is deliberately left unwritten past each chunk's valid rows, so those
|
||||
# elements are whatever the allocator last held.
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
def _through_kernels(q, k, v, conv: dict | None):
|
||||
ws = fused_kda_prologue(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp["raw_g"],
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
dt_bias=inp["dt_bias"],
|
||||
scale=scale,
|
||||
lower_bound=LOWER_BOUND,
|
||||
cu_seqlens=inp["cu"],
|
||||
**(conv or {}),
|
||||
)
|
||||
return fused_kda_chunk(
|
||||
qg=ws["qg"],
|
||||
w=ws["w"],
|
||||
u=ws["u"],
|
||||
kg_t=ws["kg_t"],
|
||||
aqk=ws["aqk"],
|
||||
decay=ws["decay"],
|
||||
out=torch.empty_like(ws["u"]),
|
||||
scale=scale,
|
||||
cu_seqlens=inp["cu"],
|
||||
initial_state=inp["h0"].clone(),
|
||||
output_final_state=True,
|
||||
)
|
||||
|
||||
qr, kr, vr = _bands(conv_ref, True)
|
||||
o_ref, s_ref = _through_kernels(qr, kr, vr, None)
|
||||
|
||||
got_state = state.clone()
|
||||
qf, kf, vf = _bands(mixed, False)
|
||||
o_got, s_got = _through_kernels(
|
||||
qf,
|
||||
kf,
|
||||
vf,
|
||||
dict(
|
||||
conv_weight=weight,
|
||||
conv_state=got_state,
|
||||
conv_state_indices=idx,
|
||||
conv_has_initial_state=has_init,
|
||||
),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(s_got, s_ref, rtol=1e-3, atol=1e-3)
|
||||
torch.testing.assert_close(
|
||||
got_state.transpose(-1, -2).float(), state_ref.float(), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4, 8])
|
||||
@pytest.mark.parametrize("seqlens", [[64], [64, 64], [100, 64], [513, 64, 1]])
|
||||
def test_chunk_group_scan_matches_serial_walk(
|
||||
groups: int, seqlens: list[int], monkeypatch
|
||||
) -> None:
|
||||
"""Splitting the chunk walk into parallel groups must not change the result.
|
||||
|
||||
Two bugs lived here, both invisible while the path was gated to >= 128
|
||||
chunks per sequence:
|
||||
|
||||
* a group with no chunks returned before writing its ``B_g``, so the scan
|
||||
folded whatever the workspace last held into the final state — these
|
||||
shapes have fewer chunks than groups, so most groups are empty;
|
||||
* the second pass started every group after the first from a zero state
|
||||
instead of the one the scan composed, which only looked right because a
|
||||
strongly decaying gate makes the dropped term small.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs(seqlens, seed=5 + groups)
|
||||
_force_groups(monkeypatch, 1)
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
_force_groups(monkeypatch, groups)
|
||||
o_got, state_got = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seqlens", [[320], [513, 64, 1, 1200], [64, 64, 64]])
|
||||
def test_accepts_chunk_metadata_from_the_attention_builder(
|
||||
seqlens: list[int],
|
||||
) -> None:
|
||||
"""The layer passes precomputed chunk metadata, not None.
|
||||
|
||||
``KimiK3ROCmKDAMetadataBuilder`` builds these on device to avoid a blocking
|
||||
device->host copy. Passing ``None`` instead takes a different path that
|
||||
builds them locally, so drive the op with the builder's own tensors.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import prepare_chunk_metadata_device
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
|
||||
inp = _inputs(seqlens, seed=17)
|
||||
cu = inp["cu"]
|
||||
chunk_indices, chunk_offsets = prepare_chunk_metadata_device(
|
||||
cu, cu.cpu(), FLA_CHUNK_SIZE
|
||||
)
|
||||
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
o_got, state_got = _run(
|
||||
inp, use_fused=True, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def _checkpoint_buffer(rows: int) -> torch.Tensor:
|
||||
"""A destination filled with NaN, so an unwritten row is unmistakable."""
|
||||
return torch.full(
|
||||
(rows, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def _prefix_inputs(inp: dict, seq: int, num_tokens: int) -> dict:
|
||||
"""Sequence ``seq`` truncated to its first ``num_tokens`` tokens."""
|
||||
bos = int(inp["cu"][seq])
|
||||
tokens = slice(bos, bos + num_tokens)
|
||||
prefix = dict(inp)
|
||||
for key in ("q", "k", "v", "raw_g", "raw_beta"):
|
||||
prefix[key] = inp[key][:, tokens].contiguous()
|
||||
prefix["h0"] = inp["h0"][seq : seq + 1].contiguous()
|
||||
prefix["cu"] = torch.tensor([0, num_tokens], device="cuda", dtype=torch.int32)
|
||||
return prefix
|
||||
|
||||
|
||||
def _run_with_checkpoints(
|
||||
inp: dict, offsets: list[int], rows: int | None = None, **kw
|
||||
) -> tuple[torch.Tensor, tuple]:
|
||||
ckpt = _checkpoint_buffer(len(offsets) if rows is None else rows)
|
||||
result = _run(
|
||||
inp,
|
||||
use_fused=True,
|
||||
checkpoint_state=ckpt,
|
||||
checkpoint_offsets=torch.tensor(offsets, device="cuda", dtype=torch.int32),
|
||||
**kw,
|
||||
)
|
||||
return ckpt, result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens,offsets",
|
||||
[
|
||||
([256], [64]), # the first interior boundary
|
||||
([256], [192]), # the last one
|
||||
([1024], [512]),
|
||||
([320, 320], [64, 256]), # every sequence checkpoints
|
||||
([513, 64, 1, 1200], [448, 0, 0, 640]), # ragged, most opted out
|
||||
],
|
||||
)
|
||||
def test_checkpoint_matches_a_prefill_that_stops_at_the_offset(
|
||||
seqlens: list[int], offsets: list[int], monkeypatch
|
||||
) -> None:
|
||||
"""A checkpoint is only useful if resuming from it is indistinguishable.
|
||||
|
||||
The scheduler serves the exported state on a later prefix-cache hit as if
|
||||
the prompt had been prefilled up to that block and no further, so the
|
||||
kernel has to hand back exactly what a prefill truncated there produces --
|
||||
not merely something close. The walk passes through that state anyway, so
|
||||
equality is bit-exact and the test asserts it as such.
|
||||
|
||||
Sequences with offset 0 opt out, and their rows must stay untouched.
|
||||
"""
|
||||
_requires_kernel()
|
||||
_force_groups(monkeypatch, 1)
|
||||
|
||||
inp = _inputs(seqlens, seed=23 + len(seqlens))
|
||||
ckpt, _ = _run_with_checkpoints(inp, offsets)
|
||||
|
||||
for i, offset in enumerate(offsets):
|
||||
if offset == 0:
|
||||
assert ckpt[i].isnan().all()
|
||||
continue
|
||||
_, truncated = _run(_prefix_inputs(inp, i, offset), use_fused=True)
|
||||
torch.testing.assert_close(ckpt[i], truncated[0], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_checkpoint_leaves_the_output_and_final_state_alone() -> None:
|
||||
"""Exporting a checkpoint must not perturb the walk it is taken from."""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [768, 320]
|
||||
inp = _inputs(seqlens, seed=29)
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
_, (o_got, state_got) = _run_with_checkpoints(inp, [512, 128])
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=0, atol=0)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [100, 63, 65, 1024])
|
||||
def test_unusable_checkpoint_offsets_write_nothing(offset: int) -> None:
|
||||
"""Offsets the walk never reaches must leave the destination untouched.
|
||||
|
||||
The recurrent state only exists at chunk boundaries, so an offset that is
|
||||
not a multiple of ``KDA_CHECKPOINT_ALIGNMENT`` -- or that lies past the end
|
||||
of the sequence -- cannot be honoured. Rounding it to a nearby boundary
|
||||
would hand the scheduler a state for the wrong token count, so the kernel
|
||||
stores nothing and leaves the caller to filter these out before the block
|
||||
is hashed.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import KDA_CHECKPOINT_ALIGNMENT
|
||||
|
||||
assert offset % KDA_CHECKPOINT_ALIGNMENT != 0 or offset > 512
|
||||
inp = _inputs([512], seed=31)
|
||||
ckpt, _ = _run_with_checkpoints(inp, [offset])
|
||||
|
||||
assert ckpt.isnan().all()
|
||||
|
||||
|
||||
def test_checkpoint_state_indices_pick_the_destination_row() -> None:
|
||||
"""With indices the export lands straight in the paged state cache.
|
||||
|
||||
FlashKDA writes one row per sequence into a staging buffer that the caller
|
||||
then scatters. The ROCm walk holds the state in registers already, so it
|
||||
can be told which cache block each sequence owns and skip the staging
|
||||
buffer and its scatter launch entirely. A negative row opts a sequence out,
|
||||
the same way a zero offset does.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [256, 256, 256]
|
||||
rows = 8
|
||||
inp = _inputs(seqlens, seed=37)
|
||||
indices = [5, -1, 2]
|
||||
ckpt, _ = _run_with_checkpoints(
|
||||
inp,
|
||||
[64, 64, 128],
|
||||
rows=rows,
|
||||
checkpoint_state_indices=torch.tensor(
|
||||
indices, device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
written = {5: (0, 64), 2: (2, 128)}
|
||||
for row in range(rows):
|
||||
if row not in written:
|
||||
assert ckpt[row].isnan().all(), f"row {row} was written"
|
||||
continue
|
||||
seq, offset = written[row]
|
||||
_, truncated = _run(_prefix_inputs(inp, seq, offset), use_fused=True)
|
||||
torch.testing.assert_close(ckpt[row], truncated[0], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
def test_checkpoint_survives_the_group_split(groups: int, monkeypatch) -> None:
|
||||
"""The split walks each group from a composed state, not from zero.
|
||||
|
||||
Only the second pass holds the true state at an interior boundary, so a
|
||||
checkpoint taken from the first pass would be the group-local state and
|
||||
silently wrong.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs([1024], seed=41)
|
||||
_force_groups(monkeypatch, 1)
|
||||
_, serial = _run(_prefix_inputs(inp, 0, 512), use_fused=True)
|
||||
_force_groups(monkeypatch, groups)
|
||||
ckpt, _ = _run_with_checkpoints(inp, [512])
|
||||
|
||||
torch.testing.assert_close(ckpt[0], serial[0], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def test_triton_fallback_refuses_a_checkpoint_export() -> None:
|
||||
"""Dropping the export silently would serve an unwritten state.
|
||||
|
||||
The scheduler allocates and hashes the checkpoint block on the strength of
|
||||
the layer opting in, so a backend that cannot write it has to say so rather
|
||||
than return a plausible output.
|
||||
"""
|
||||
inp = _inputs([256], seed=43)
|
||||
with pytest.raises(NotImplementedError):
|
||||
_run(
|
||||
inp,
|
||||
use_fused=False,
|
||||
checkpoint_state=_checkpoint_buffer(1),
|
||||
checkpoint_offsets=torch.tensor([64], device="cuda", dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paged recurrent state
|
||||
#
|
||||
# Each sequence reads its initial state from a row of the paged cache and
|
||||
# writes its final state back to the same row. These check that the result
|
||||
# matches the dense initial_state/final_state pair, that a sequence marked
|
||||
# cold starts from zeros rather than from whatever its row held, and that rows
|
||||
# no sequence owns are left untouched.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _paged_cache(inp: dict, warm: list[bool], rows: list[int], slots: int):
|
||||
"""A cache holding each sequence's h0 at its row, a fill pattern elsewhere."""
|
||||
cache = torch.full(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
-7.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
|
||||
for n, row in enumerate(rows):
|
||||
# A cold row keeps its fill pattern, which must not reach the walk.
|
||||
if warm[n]:
|
||||
cache[row] = inp["h0"][n]
|
||||
return cache, idx, torch.tensor(warm, device="cuda", dtype=torch.bool)
|
||||
|
||||
|
||||
def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
|
||||
return chunk_kda_prefill(
|
||||
q=inp["q"].clone(),
|
||||
k=inp["k"].clone(),
|
||||
v=inp["v"].clone(),
|
||||
raw_g=inp["raw_g"].clone(),
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
g_bias=inp["dt_bias"],
|
||||
lower_bound=LOWER_BOUND,
|
||||
state_cache=cache,
|
||||
state_indices=idx,
|
||||
has_initial_state=warm,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
backend="fused" if use_fused else "triton",
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
@pytest.mark.parametrize("seqlens", [[512], [2048], [320, 64], [513, 64, 1]])
|
||||
def test_paged_state_matches_the_dense_pair(
|
||||
use_fused: bool, seqlens: list[int]
|
||||
) -> None:
|
||||
"""A paged cache must give the same result as the dense pair."""
|
||||
_requires_kernel()
|
||||
inp = _inputs(seqlens, seed=11)
|
||||
n = len(seqlens)
|
||||
rows = [3 * i + 5 for i in range(n)]
|
||||
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
|
||||
before = cache.clone()
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=use_fused)
|
||||
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
assert ht_got is None, "the final state is in the cache, not returned"
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
untouched = torch.ones(cache.shape[0], dtype=torch.bool, device="cuda")
|
||||
untouched[idx.long()] = False
|
||||
assert torch.equal(cache[untouched], before[untouched]), (
|
||||
"a row no sequence owns was written"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
|
||||
"""`has_initial_state=False` must ignore whatever the row already holds."""
|
||||
_requires_kernel()
|
||||
seqlens = [256, 256, 256]
|
||||
inp = _inputs(seqlens, seed=12)
|
||||
warm = [True, False, True]
|
||||
rows = [4, 9, 1]
|
||||
cache, idx, warm_t = _paged_cache(inp, warm, rows, slots=16)
|
||||
|
||||
# The dense reference zeroes exactly the cold sequences' initial states.
|
||||
ref = dict(inp)
|
||||
ref["h0"] = inp["h0"].clone()
|
||||
ref["h0"][1] = 0.0
|
||||
o_ref, ht_ref = _run(ref, use_fused=use_fused)
|
||||
o_got, _ = _run_paged(inp, cache, idx, warm_t, use_fused=use_fused)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None:
|
||||
# The layer passes block_table[:, 0], which is contiguous only while the
|
||||
# table has one column. Speculative decoding reserves a state page per
|
||||
# draft token, so the table widens and that column becomes strided.
|
||||
_requires_kernel()
|
||||
seqlens = [512, 320, 64]
|
||||
inp = _inputs(seqlens, seed=43)
|
||||
n = len(seqlens)
|
||||
rows = [3 * i + 5 for i in range(n)]
|
||||
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
|
||||
ref_cache = cache.clone()
|
||||
|
||||
table = torch.zeros(n, 9, dtype=torch.int32, device="cuda")
|
||||
table[:, 0] = idx
|
||||
column = table[:, 0]
|
||||
assert not column.is_contiguous()
|
||||
|
||||
o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused)
|
||||
o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref)
|
||||
torch.testing.assert_close(cache, ref_cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None:
|
||||
# A batch carrying speculative decodes pads its non-spec group with
|
||||
# zero-length sequences whose cache row still belongs to a request, so
|
||||
# consuming no tokens has to mean writing nothing and perturbing nobody.
|
||||
_requires_kernel()
|
||||
live = [200, 1, 128]
|
||||
# _inputs draws h0 last, so the padded batch's live rows are the same draw
|
||||
# as the unpadded one's and the two runs compare elementwise.
|
||||
padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41)
|
||||
rows = [3 * i + 5 for i in range(4)]
|
||||
|
||||
cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24)
|
||||
before = cache.clone()
|
||||
_run_paged(padded, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24)
|
||||
_run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused)
|
||||
|
||||
assert torch.equal(cache[rows[3]], before[rows[3]]), (
|
||||
"the padded sequence's cache row was written"
|
||||
)
|
||||
torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
|
||||
"""At G>1 the read and the write are different blocks of one launch."""
|
||||
_requires_kernel()
|
||||
_force_groups(monkeypatch, groups)
|
||||
inp = _inputs([1024, 512], seed=13)
|
||||
rows = [6, 2]
|
||||
cache, idx, warm = _paged_cache(inp, [True, True], rows, slots=12)
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
o_got, _ = _run_paged(inp, cache, idx, warm, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
def test_paged_state_and_checkpoint_export_together() -> None:
|
||||
"""The checkpoint lands in its own row while the walk owns the state row."""
|
||||
_requires_kernel()
|
||||
inp = _inputs([1024], seed=14)
|
||||
cache, idx, warm = _paged_cache(inp, [True], [3], slots=12)
|
||||
ckidx = torch.tensor([8], device="cuda", dtype=torch.int32)
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
prefix = _prefix_inputs(inp, 0, 512)
|
||||
_, ck_ref = _run(prefix, use_fused=True)
|
||||
|
||||
o_got, _ = _run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
checkpoint_state=cache,
|
||||
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
|
||||
checkpoint_state_indices=ckidx,
|
||||
)
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[ckidx.long()], ck_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
def test_paged_state_rejects_a_dense_initial_state() -> None:
|
||||
"""The two ways of passing the state are mutually exclusive."""
|
||||
_requires_kernel()
|
||||
inp = _inputs([256], seed=15)
|
||||
cache, idx, warm = _paged_cache(inp, [True], [1], slots=4)
|
||||
with pytest.raises(ValueError, match="replaces initial_state"):
|
||||
_run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
initial_state=inp["h0"].clone(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_padded_cache_stride_is_honoured(use_fused: bool) -> None:
|
||||
"""A mamba page may pad stride(0); only each state row is dense.
|
||||
|
||||
The real `recurrent_state` is not a contiguous tensor, so a kernel that
|
||||
assumes a packed ``H * V * K`` row stride addresses the wrong rows. The
|
||||
other cases in this file all build a contiguous cache, which cannot
|
||||
exercise that.
|
||||
"""
|
||||
_requires_kernel()
|
||||
seqlens = [1024, 512]
|
||||
inp = _inputs(seqlens, seed=21)
|
||||
n = len(seqlens)
|
||||
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
|
||||
slots, pad = 16, 512
|
||||
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
|
||||
cache = raw.as_strided(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
assert not cache.is_contiguous() and cache[0].is_contiguous()
|
||||
|
||||
rows = [11, 3]
|
||||
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
|
||||
warm = torch.ones(n, dtype=torch.bool, device="cuda")
|
||||
for i, r in enumerate(rows):
|
||||
cache[r] = inp["h0"][i]
|
||||
before = raw.clone()
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=use_fused)
|
||||
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
assert ht_got is None
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
# The padding between rows, and every unowned row, must be untouched.
|
||||
untouched = torch.ones(slots, dtype=torch.bool, device="cuda")
|
||||
untouched[idx.long()] = False
|
||||
assert torch.equal(raw[untouched], before[untouched])
|
||||
assert torch.equal(raw[idx.long(), row:], before[idx.long(), row:]), (
|
||||
"the kernel wrote into the inter-row padding"
|
||||
)
|
||||
|
||||
|
||||
def test_padded_checkpoint_stride_is_honoured() -> None:
|
||||
"""The checkpoint destination is the paged cache, which pads stride(0).
|
||||
|
||||
Every other checkpoint case here builds a contiguous buffer, so none of
|
||||
them would catch a kernel that assumes a packed ``H * V * K`` row stride
|
||||
and writes the snapshot into a neighbouring block.
|
||||
"""
|
||||
_requires_kernel()
|
||||
seqlens = [1024]
|
||||
inp = _inputs(seqlens, seed=31)
|
||||
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
|
||||
slots, pad = 16, 512
|
||||
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
|
||||
cache = raw.as_strided(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
assert not cache.is_contiguous()
|
||||
|
||||
state_rows = [11]
|
||||
idx = torch.tensor(state_rows, device="cuda", dtype=torch.int32)
|
||||
warm = torch.ones(1, dtype=torch.bool, device="cuda")
|
||||
cache[state_rows[0]] = inp["h0"][0]
|
||||
ck_rows = [4]
|
||||
before = raw.clone()
|
||||
|
||||
prefix = _prefix_inputs(inp, 0, 512)
|
||||
_, ck_ref = _run(prefix, use_fused=True)
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
|
||||
o_got, _ = _run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
checkpoint_state=cache,
|
||||
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
|
||||
checkpoint_state_indices=torch.tensor(
|
||||
ck_rows, device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[ck_rows[0]], ck_ref[0], rtol=5e-3, atol=5e-3)
|
||||
|
||||
touched = torch.zeros(slots, dtype=torch.bool, device="cuda")
|
||||
touched[state_rows + ck_rows] = True
|
||||
assert torch.equal(raw[~touched], before[~touched])
|
||||
assert torch.equal(raw[touched, row:], before[touched, row:]), (
|
||||
"the kernel wrote into the inter-row padding"
|
||||
)
|
||||
@@ -1,659 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm fused KDA decode kernels must match the Triton chain they replace.
|
||||
|
||||
Two fused paths fold the packed causal conv1d update, the gated delta-rule
|
||||
recurrence and the gated output RMSNorm into one launch, updating the conv
|
||||
state and the recurrent state in place:
|
||||
|
||||
* the vLLM HIP kernel ``ops.fused_kda_decode`` (gfx942 / gfx950), and
|
||||
* AITER's Gluon kernels behind ``rocm_aiter_ops`` (CDNA4+): the packed
|
||||
non-spec decode and the speculative-decode ``fused_recurrent_kda``.
|
||||
|
||||
Every output of each is compared against the unfused path the AMD layer falls
|
||||
back to, with the argument layout the layer itself uses.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _hip_kernel_arch() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
|
||||
return on_gfx950() or on_gfx942()
|
||||
|
||||
|
||||
def _aiter_gluon_kda_available() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return get_cdna_version() >= 4 and rocm_aiter_ops._kda_gluon_decode_importable()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-only KDA decode kernels"
|
||||
)
|
||||
requires_hip_kernel_arch = pytest.mark.skipif(
|
||||
not _hip_kernel_arch(),
|
||||
reason="The fused KDA decode kernel is only built for gfx942 / gfx950",
|
||||
)
|
||||
requires_aiter_gluon_kda = pytest.mark.skipif(
|
||||
not _aiter_gluon_kda_available(),
|
||||
reason="AITER Gluon KDA decode needs CDNA4+ and an AITER build that ships it",
|
||||
)
|
||||
|
||||
# Kimi-K3 KDA: 96 heads x 128, conv width 4, gate_lower_bound -5.0.
|
||||
HEAD_DIM = 128
|
||||
CONV_WIDTH = 4
|
||||
GATE_LOWER_BOUND = -5.0
|
||||
NORM_EPS = 1e-5
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _requires_kernel() -> None:
|
||||
if not hasattr(torch.ops._C, "fused_kda_decode"):
|
||||
pytest.skip("vLLM was built without the fused KDA decode kernel")
|
||||
|
||||
|
||||
class KdaDecodeInputs:
|
||||
"""One decode step of a Kimi-K3 KDA layer, in the layer's own layout.
|
||||
|
||||
``num_spec`` > 0 sizes the conv state for speculative decoding
|
||||
(``conv_width - 1 + num_spec`` taps, as ``kda_state_shape`` does).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
num_slots: int,
|
||||
seed: int = 0,
|
||||
num_spec: int = 0,
|
||||
) -> None:
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
dim = num_heads * HEAD_DIM
|
||||
self.num_heads = num_heads
|
||||
self.dim = dim
|
||||
|
||||
self.mixed_qkv = (
|
||||
torch.randn(num_tokens, 3 * dim, device=device, dtype=DTYPE) * 0.5
|
||||
)
|
||||
# conv1d weight as the layer holds it: [3 * dim, width] fp32.
|
||||
self.conv_weights = (
|
||||
torch.randn(3 * dim, CONV_WIDTH, device=device, dtype=torch.float32) * 0.3
|
||||
)
|
||||
# Width-major fp32 copy the fused kernel indexes: [3, width, dim].
|
||||
self.decode_conv1d_weight = torch.stack(
|
||||
[
|
||||
self.conv_weights[i * dim : (i + 1) * dim].transpose(0, 1).contiguous()
|
||||
for i in range(3)
|
||||
]
|
||||
)
|
||||
# SD cache layout: [slots, taps, 3 * dim]; the layer transposes it.
|
||||
self.conv_state = (
|
||||
torch.randn(
|
||||
num_slots,
|
||||
CONV_WIDTH - 1 + num_spec,
|
||||
3 * dim,
|
||||
device=device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
self.recurrent_state = (
|
||||
torch.randn(
|
||||
num_slots,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
HEAD_DIM,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
* 0.1
|
||||
)
|
||||
self.g1 = (
|
||||
torch.randn(1, num_tokens, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
|
||||
* 0.5
|
||||
)
|
||||
self.g2 = (
|
||||
torch.randn(num_tokens, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
|
||||
* 0.5
|
||||
)
|
||||
self.beta = torch.randn(1, num_tokens, num_heads, device=device, dtype=DTYPE)
|
||||
self.A_log = torch.randn(num_heads, device=device, dtype=torch.float32) * 0.5
|
||||
self.dt_bias = torch.randn(dim, device=device, dtype=torch.float32) * 0.1
|
||||
self.norm_weight_bf16 = 1 + 0.1 * torch.randn(
|
||||
HEAD_DIM, device=device, dtype=DTYPE
|
||||
)
|
||||
self.decode_norm_weight = self.norm_weight_bf16.float()
|
||||
# Distinct, shuffled slots: the kernel must honour the indirection.
|
||||
# Slot 0 is NULL_BLOCK_ID and never allocated for live state, and the
|
||||
# Triton reference short-circuits it, so draw from [1, num_slots).
|
||||
assert num_slots > num_tokens, "need a spare slot to detect stray writes"
|
||||
self.state_indices = (
|
||||
torch.randperm(num_slots - 1, device=device)[:num_tokens] + 1
|
||||
).to(torch.int32)
|
||||
|
||||
def conv_state_view(self, state: torch.Tensor) -> torch.Tensor:
|
||||
return state.transpose(-1, -2)
|
||||
|
||||
def empty_out(self) -> torch.Tensor:
|
||||
num_tokens = self.mixed_qkv.shape[0]
|
||||
return torch.empty(
|
||||
1, num_tokens, self.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _gated_rmsnorm(
|
||||
x: torch.Tensor, gate: torch.Tensor, weight: torch.Tensor, eps: float
|
||||
) -> torch.Tensor:
|
||||
x_float = x.float()
|
||||
variance = x_float.pow(2).mean(dim=-1, keepdim=True)
|
||||
normed = x_float * torch.rsqrt(variance + eps) * weight.float()
|
||||
return (normed * torch.sigmoid(gate.float())).to(x.dtype)
|
||||
|
||||
|
||||
def _run_triton_chain(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""conv1d update -> recurrent decode -> gated norm, as the layer runs it."""
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
conv_out = torch.empty_like(inp.mixed_qkv)
|
||||
causal_conv1d_update(
|
||||
inp.mixed_qkv,
|
||||
inp.conv_state_view(conv_state),
|
||||
inp.conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_state_indices=inp.state_indices,
|
||||
validate_data=True,
|
||||
out=conv_out,
|
||||
)
|
||||
core_attn_out, _ = fused_recurrent_kda_packed_decode(
|
||||
mixed_qkv=conv_out,
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
state_indices=inp.state_indices,
|
||||
)
|
||||
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
def _run_fused(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
num_tokens = inp.mixed_qkv.shape[0]
|
||||
out = torch.empty(
|
||||
1,
|
||||
num_tokens,
|
||||
inp.num_heads,
|
||||
HEAD_DIM,
|
||||
device=inp.mixed_qkv.device,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
ops.fused_kda_decode(
|
||||
x=inp.mixed_qkv,
|
||||
weight=inp.decode_conv1d_weight,
|
||||
bias=None,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
state_indices=inp.state_indices,
|
||||
state=recurrent_state,
|
||||
out=out,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
output_gate=inp.g2,
|
||||
norm_weight=inp.decode_norm_weight,
|
||||
norm_eps=NORM_EPS,
|
||||
)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@pytest.mark.parametrize("num_heads", [12, 24, 96])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_matches_triton_chain(num_heads: int, num_tokens: int) -> None:
|
||||
_requires_kernel()
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_fused(inp)
|
||||
|
||||
# The fused kernel keeps the recurrent output in fp32 through the norm,
|
||||
# while the Triton chain rounds it to BF16 in between.
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
# The conv state is a pure shift-and-append: it must be bit-exact.
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by state_indices may be written."""
|
||||
_requires_kernel()
|
||||
num_tokens, num_heads, num_slots = 4, 12, 9
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
|
||||
|
||||
_, actual_conv, actual_state = _run_fused(inp)
|
||||
|
||||
touched = set(inp.state_indices.tolist())
|
||||
untouched = [slot for slot in range(num_slots) if slot not in touched]
|
||||
assert untouched, "test needs at least one unused slot"
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_without_output_norm() -> None:
|
||||
"""Omitting the gate/norm pair returns the raw recurrent output."""
|
||||
_requires_kernel()
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
inp = KdaDecodeInputs(5, 12, num_slots=8, seed=7)
|
||||
_, _, expected_state = _run_triton_chain(inp)
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
out = torch.empty(1, 5, inp.num_heads, HEAD_DIM, device="cuda", dtype=DTYPE)
|
||||
ops.fused_kda_decode(
|
||||
x=inp.mixed_qkv,
|
||||
weight=inp.decode_conv1d_weight,
|
||||
bias=None,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
state_indices=inp.state_indices,
|
||||
state=recurrent_state,
|
||||
out=out,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(recurrent_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
assert torch.isfinite(out).all()
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
@torch.inference_mode()
|
||||
def test_fused_kda_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
|
||||
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0, and
|
||||
the Triton chain zeroes those rows' output while leaving slot 0 alone. The
|
||||
fused kernel must do the same rather than read-modify-write slot 0's state
|
||||
once per padded row.
|
||||
"""
|
||||
_requires_kernel()
|
||||
num_real, num_padded, num_heads = 3, 5, 12
|
||||
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
|
||||
inp.state_indices[num_real:] = 0
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_fused(inp)
|
||||
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
assert not actual_out[0, num_real:].any(), "padded rows must produce zeros"
|
||||
# Slot 0 must be untouched by both paths.
|
||||
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_hip_kernel_arch
|
||||
def test_decode_weights_staged_after_loading(dist_init) -> None:
|
||||
"""The fused-decode weight copies must not depend on the weight loaders.
|
||||
|
||||
Loaders that bypass ``weight_loader`` (e.g. ``--load-format dummy``) left
|
||||
them uninitialized, so decode could read NaN. Loading runs with grad mode
|
||||
on, so the copies must also stay out of autograd.
|
||||
"""
|
||||
_requires_kernel()
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.model_loader.utils import process_weights_after_loading
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
|
||||
config = KimiLinearConfig(
|
||||
hidden_size=256,
|
||||
linear_attn_config={
|
||||
"kda_layers": [1],
|
||||
"full_attn_layers": [],
|
||||
"head_dim": HEAD_DIM,
|
||||
"num_heads": 12,
|
||||
"short_conv_kernel_size": CONV_WIDTH,
|
||||
"use_full_rank_gate": True,
|
||||
"gate_lower_bound": GATE_LOWER_BOUND,
|
||||
},
|
||||
)
|
||||
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B", dtype=DTYPE)
|
||||
with set_current_vllm_config(vllm_config), torch.device("cuda"):
|
||||
layer = KimiK3DeltaAttention(
|
||||
config, vllm_config, prefix="model.layers.0.self_attn"
|
||||
)
|
||||
assert layer.decode_conv1d_weight is not None
|
||||
# The AITER Gluon decode path takes o_norm.weight directly, so only the HIP
|
||||
# kernel gets an fp32 norm copy.
|
||||
assert (layer.decode_norm_weight is None) == layer.use_aiter_decode
|
||||
with torch.no_grad():
|
||||
for param in layer.parameters():
|
||||
param.normal_()
|
||||
layer.decode_conv1d_weight.fill_(float("nan"))
|
||||
if layer.decode_norm_weight is not None:
|
||||
layer.decode_norm_weight.fill_(float("nan"))
|
||||
|
||||
process_weights_after_loading(
|
||||
torch.nn.ModuleList([layer]), vllm_config.model_config, torch.device("cuda")
|
||||
)
|
||||
|
||||
dim = layer.local_projection_size
|
||||
conv_weights = layer.conv1d.weight.squeeze(1)
|
||||
expected_conv = torch.stack(
|
||||
[conv_weights[i * dim : (i + 1) * dim].transpose(0, 1) for i in range(3)]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
layer.decode_conv1d_weight, expected_conv, atol=0, rtol=0
|
||||
)
|
||||
if layer.decode_norm_weight is not None:
|
||||
torch.testing.assert_close(
|
||||
layer.decode_norm_weight, layer.o_norm.weight.float(), atol=0, rtol=0
|
||||
)
|
||||
assert not layer.decode_norm_weight.requires_grad
|
||||
assert not layer.decode_conv1d_weight.requires_grad
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# non-spec decode: fused_recurrent_kda_packed_decode
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _run_aiter_packed_decode(
|
||||
inp: KdaDecodeInputs,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for decode batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
num_tokens = inp.mixed_qkv.shape[0]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
|
||||
mixed_qkv=inp.mixed_qkv,
|
||||
g=inp.g1[0],
|
||||
beta=inp.beta[0],
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
ssm_state_indices=inp.state_indices[:num_tokens],
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:num_tokens],
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[0, :num_tokens],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 128])
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_matches_triton_chain(
|
||||
num_heads: int, num_tokens: int
|
||||
) -> None:
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots=max(num_tokens, 4) + 3)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
# The fused kernel keeps the recurrent output in fp32 through the norm,
|
||||
# while the Triton chain rounds it to BF16 in between.
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
# The conv state is a pure shift-and-append: it must be bit-exact.
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_leaves_untouched_slots_alone() -> None:
|
||||
"""Only the slots named by the state indices may be written."""
|
||||
num_tokens, num_heads, num_slots = 4, 12, 9
|
||||
inp = KdaDecodeInputs(num_tokens, num_heads, num_slots, seed=3)
|
||||
|
||||
_, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
touched = set(inp.state_indices.tolist())
|
||||
untouched = [slot for slot in range(num_slots) if slot not in touched]
|
||||
assert untouched, "test needs at least one unused slot"
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@torch.inference_mode()
|
||||
def test_aiter_packed_decode_skips_null_block_padding() -> None:
|
||||
"""A CUDA-graph decode batch is padded with NULL_BLOCK_ID (0).
|
||||
|
||||
gdn_attn.py fills the tail of ``non_spec_state_indices_tensor`` with 0 and
|
||||
the Triton chain leaves slot 0 alone; the fused kernel's pad_slot_guard
|
||||
must do the same rather than read-modify-write slot 0 once per padded row.
|
||||
"""
|
||||
num_real, num_padded, num_heads = 3, 5, 12
|
||||
inp = KdaDecodeInputs(num_real + num_padded, num_heads, num_slots=12, seed=11)
|
||||
inp.state_indices[num_real:] = 0
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_triton_chain(inp)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_packed_decode(inp)
|
||||
|
||||
torch.testing.assert_close(
|
||||
actual_out[0, :num_real], expected_out[0, :num_real], atol=3e-2, rtol=3e-2
|
||||
)
|
||||
assert torch.isfinite(actual_out).all()
|
||||
# Slot 0 must be untouched by both paths.
|
||||
torch.testing.assert_close(actual_conv[0], inp.conv_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state[0], inp.recurrent_state[0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=2e-3)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# spec-only decode: fused_recurrent_kda with conv / gate / norm fused in
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SpecBatch:
|
||||
"""A spec-only KDA batch: ``num_spec + 1`` query tokens per request at most.
|
||||
|
||||
``seq_lens`` are the per-request query lengths, ``num_accepted`` the tokens
|
||||
verified in the previous step; ``state_indices`` is the [N, num_spec + 1]
|
||||
per-token slot table the GDN metadata hands the layer.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_lens: list[int],
|
||||
num_accepted: list[int],
|
||||
num_heads: int,
|
||||
num_spec: int,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
assert len(seq_lens) == len(num_accepted)
|
||||
assert max(seq_lens) <= num_spec + 1
|
||||
n_seqs = len(seq_lens)
|
||||
slots_per_seq = num_spec + 1
|
||||
num_slots = n_seqs * slots_per_seq + 2
|
||||
self.inp = KdaDecodeInputs(
|
||||
sum(seq_lens), num_heads, num_slots, seed=seed, num_spec=num_spec
|
||||
)
|
||||
# Distinct slots per (request, token), shuffled, none at slot 0.
|
||||
perm = torch.randperm(num_slots - 1, device="cuda")[: n_seqs * slots_per_seq]
|
||||
self.state_indices = (perm + 1).to(torch.int32).view(n_seqs, slots_per_seq)
|
||||
self.query_start_loc = torch.tensor(
|
||||
[0] + torch.tensor(seq_lens).cumsum(0).tolist(),
|
||||
dtype=torch.int32,
|
||||
device="cuda",
|
||||
)
|
||||
self.num_accepted_tokens = torch.tensor(
|
||||
num_accepted, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
self.max_query_len = slots_per_seq
|
||||
|
||||
|
||||
def _run_spec_triton_chain(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The layer's unfused spec path: conv update -> recurrent KDA -> norm."""
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import fused_recurrent_kda
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
conv_out = causal_conv1d_update(
|
||||
inp.mixed_qkv,
|
||||
inp.conv_state_view(conv_state),
|
||||
inp.conv_weights,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_state_indices=sb.state_indices[:, 0],
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
query_start_loc=sb.query_start_loc,
|
||||
max_query_len=sb.max_query_len,
|
||||
validate_data=False,
|
||||
out=torch.empty_like(inp.mixed_qkv),
|
||||
)
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM) for x in conv_out.split(inp.dim, dim=-1)
|
||||
)
|
||||
core_attn_out, _ = fused_recurrent_kda(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp.g1,
|
||||
raw_beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
)
|
||||
out = _gated_rmsnorm(core_attn_out, inp.g2, inp.norm_weight_bf16, NORM_EPS)
|
||||
return out, conv_state, recurrent_state
|
||||
|
||||
|
||||
def _run_aiter_spec(
|
||||
sb: SpecBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Exactly the call KimiK3DeltaAttention._forward makes for spec-only batches."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
inp = sb.inp
|
||||
conv_state = inp.conv_state.clone()
|
||||
recurrent_state = inp.recurrent_state.clone()
|
||||
q, k, v = (
|
||||
x.view(1, -1, inp.num_heads, HEAD_DIM)
|
||||
for x in inp.mixed_qkv.split(inp.dim, dim=-1)
|
||||
)
|
||||
n_spec = q.shape[1]
|
||||
core_attn_out = inp.empty_out()
|
||||
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=inp.g1,
|
||||
beta=inp.beta,
|
||||
A_log=inp.A_log,
|
||||
dt_bias=inp.dt_bias,
|
||||
lower_bound=GATE_LOWER_BOUND,
|
||||
initial_state=recurrent_state,
|
||||
cu_seqlens=sb.query_start_loc,
|
||||
ssm_state_indices=sb.state_indices,
|
||||
num_accepted_tokens=sb.num_accepted_tokens,
|
||||
conv_state=inp.conv_state_view(conv_state),
|
||||
conv_weight=inp.decode_conv1d_weight,
|
||||
out_gate=inp.g2[:n_spec].unsqueeze(0),
|
||||
norm_weight=inp.norm_weight_bf16,
|
||||
norm_eps=NORM_EPS,
|
||||
out=core_attn_out[:, :n_spec],
|
||||
)
|
||||
return core_attn_out, conv_state, recurrent_state
|
||||
|
||||
|
||||
@requires_aiter_gluon_kda
|
||||
@pytest.mark.parametrize("num_heads", [12, 24])
|
||||
@pytest.mark.parametrize(
|
||||
"num_spec, seq_lens, num_accepted",
|
||||
[
|
||||
(3, [4, 4, 4], [1, 2, 4]), # Kimi-K3 recipe: 3 draft tokens, uniform
|
||||
(3, [4, 2, 1, 4], [4, 1, 1, 3]), # ragged after partial acceptance
|
||||
(6, [7, 3], [1, 4]), # longer window, two requests
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_aiter_spec_decode_matches_triton_chain(
|
||||
num_heads: int, num_spec: int, seq_lens: list[int], num_accepted: list[int]
|
||||
) -> None:
|
||||
sb = SpecBatch(seq_lens, num_accepted, num_heads, num_spec, seed=5)
|
||||
|
||||
expected_out, expected_conv, expected_state = _run_spec_triton_chain(sb)
|
||||
actual_out, actual_conv, actual_state = _run_aiter_spec(sb)
|
||||
|
||||
torch.testing.assert_close(actual_out, expected_out, atol=3e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
|
||||
# Nothing outside the slot table may change.
|
||||
touched = set(sb.state_indices.flatten().tolist())
|
||||
untouched = [s for s in range(sb.inp.conv_state.shape[0]) if s not in touched]
|
||||
assert untouched
|
||||
torch.testing.assert_close(
|
||||
actual_conv[untouched], sb.inp.conv_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual_state[untouched], sb.inp.recurrent_state[untouched], atol=0, rtol=0
|
||||
)
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
|
||||
from vllm.models.kimi_k3.amd.linear import KimiDecoderLayer
|
||||
|
||||
|
||||
class _ReturningAttention(nn.Module):
|
||||
def __init__(self, result: torch.Tensor) -> None:
|
||||
super().__init__()
|
||||
self.result = result
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return self.result
|
||||
|
||||
|
||||
class _TupleProjection(nn.Module):
|
||||
def __init__(self, result: torch.Tensor) -> None:
|
||||
super().__init__()
|
||||
self.result = result
|
||||
|
||||
def forward(self, _: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||
return self.result, None
|
||||
|
||||
|
||||
def _make_layer(self_attn: nn.Module) -> KimiDecoderLayer:
|
||||
layer = object.__new__(KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.self_attn = self_attn
|
||||
return layer
|
||||
|
||||
|
||||
def test_kda_self_attention_returns_projection_storage_directly() -> None:
|
||||
hidden_states = torch.randn(4, 8)
|
||||
projected = torch.randn_like(hidden_states)
|
||||
layer = _make_layer(_ReturningAttention(projected))
|
||||
|
||||
output = layer._run_self_attn(torch.arange(4), hidden_states)
|
||||
|
||||
assert output is projected
|
||||
assert output.data_ptr() == projected.data_ptr()
|
||||
|
||||
|
||||
def test_k3_kda_forward_returns_output_projection_storage() -> None:
|
||||
hidden_states = torch.randn(4, 8)
|
||||
projected = torch.randn_like(hidden_states)
|
||||
|
||||
def fill_core_output(**kwargs: torch.Tensor) -> None:
|
||||
kwargs["core_attn_out"].fill_(1)
|
||||
|
||||
attention = SimpleNamespace(
|
||||
local_projection_size=4,
|
||||
head_dim=2,
|
||||
local_num_heads=2,
|
||||
in_proj_padding=0,
|
||||
in_proj_qkvgfab=_TupleProjection(torch.randn(4, 20)),
|
||||
f_b_proj=_TupleProjection(torch.randn(4, 4)),
|
||||
o_proj=_TupleProjection(projected),
|
||||
_forward=fill_core_output,
|
||||
)
|
||||
|
||||
output = KimiK3DeltaAttention.forward(attention, hidden_states, torch.arange(4))
|
||||
|
||||
assert output is projected
|
||||
assert output.data_ptr() == projected.data_ptr()
|
||||
@@ -1,163 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="AITER fused conv1d is ROCm-only"
|
||||
)
|
||||
|
||||
WIDTH = 4 # Kimi-K3 short_conv_kernel_size
|
||||
STATE_LEN = WIDTH - 1
|
||||
HEAD_DIM = 128
|
||||
LOCAL_HEADS = 4 # per-TP-rank heads; projection = 512, packed dim = 1536
|
||||
|
||||
|
||||
def _fused_fn():
|
||||
gdn = pytest.importorskip("aiter.ops.triton.gated_delta_net")
|
||||
return gdn.causal_conv1d_split_qkv_triton_tile_fn
|
||||
|
||||
|
||||
def _make_inputs(seqlens, has_initial_state, seed=0, device="cuda"):
|
||||
torch.manual_seed(seed)
|
||||
proj = HEAD_DIM * LOCAL_HEADS
|
||||
dim = 3 * proj
|
||||
total = sum(seqlens)
|
||||
n_seqs = len(seqlens)
|
||||
# Token-major activations, as produced by the in_proj GEMM.
|
||||
mixed_qkv = torch.randn(total, dim, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn(dim, WIDTH, dtype=torch.bfloat16, device=device) * 0.3
|
||||
# More cache lines than sequences, assigned in a shuffled order. Line 0 is
|
||||
# vLLM's null block (NULL_BLOCK_ID), which causal_conv1d_fn skips, so real
|
||||
# sequences never map to it.
|
||||
n_lines = n_seqs + 2
|
||||
conv_state = torch.randn(
|
||||
n_lines, dim, STATE_LEN, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
cache_indices = (torch.randperm(n_lines - 1, device=device)[:n_seqs] + 1).to(
|
||||
torch.int32
|
||||
)
|
||||
query_start_loc = torch.tensor(
|
||||
[0] + list(torch.tensor(seqlens).cumsum(0).tolist()),
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
has_initial = torch.full(
|
||||
(n_seqs,), has_initial_state, dtype=torch.bool, device=device
|
||||
)
|
||||
return mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
|
||||
|
||||
def _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""fp32 per-sequence depthwise causal conv + SiLU.
|
||||
|
||||
Returns (out[T, dim], new_conv_state).
|
||||
"""
|
||||
x = mixed_qkv.float().t() # (dim, T)
|
||||
w = weight.float()
|
||||
dim = x.shape[0]
|
||||
out = torch.empty_like(x)
|
||||
new_state = conv_state.float().clone()
|
||||
qsl = query_start_loc.tolist()
|
||||
for s in range(len(qsl) - 1):
|
||||
start, end = qsl[s], qsl[s + 1]
|
||||
ci = int(cache_indices[s])
|
||||
xs = x[:, start:end]
|
||||
if bool(has_initial[s]):
|
||||
xin = torch.cat([conv_state[ci].float(), xs], dim=1)
|
||||
else:
|
||||
xin = F.pad(xs, (STATE_LEN, 0))
|
||||
y = F.conv1d(xin.unsqueeze(0), w.unsqueeze(1), None, groups=dim)[0]
|
||||
out[:, start:end] = F.silu(y)
|
||||
new_state[ci] = xin[:, -STATE_LEN:]
|
||||
return out.t(), new_state
|
||||
|
||||
|
||||
def _three_launch(
|
||||
mixed_qkv, weight, conv_state, cache_indices, query_start_loc, has_initial
|
||||
):
|
||||
"""The pre-fusion KDA prefill path: one causal_conv1d_fn per projection."""
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
outs = []
|
||||
for x, state, w in zip(
|
||||
mixed_qkv.split(proj, dim=-1),
|
||||
conv_state.split(proj, dim=-2),
|
||||
weight.split(proj, dim=0),
|
||||
):
|
||||
outs.append(
|
||||
causal_conv1d_fn(
|
||||
x.transpose(0, 1),
|
||||
w,
|
||||
None,
|
||||
activation="silu",
|
||||
conv_states=state,
|
||||
has_initial_state=has_initial,
|
||||
cache_indices=cache_indices,
|
||||
query_start_loc=query_start_loc,
|
||||
).transpose(0, 1)
|
||||
)
|
||||
return outs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens", [[1024], [8, 1, 300], [64, 64, 64], [513, 64, 1, 1200]]
|
||||
)
|
||||
@pytest.mark.parametrize("has_initial_state", [False, True])
|
||||
@pytest.mark.parametrize("block_m", [8, 128])
|
||||
def test_fused_split_qkv_conv1d_matches_reference(seqlens, has_initial_state, block_m):
|
||||
fused_fn = _fused_fn()
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial = _make_inputs(
|
||||
seqlens, has_initial_state
|
||||
)
|
||||
proj = mixed_qkv.shape[1] // 3
|
||||
|
||||
ref_out, ref_state = _reference(
|
||||
mixed_qkv, weight, conv_state, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_3 = conv_state.clone()
|
||||
q3, k3, v3 = _three_launch(
|
||||
mixed_qkv, weight, state_3, cache_indices, qsl, has_initial
|
||||
)
|
||||
|
||||
state_f = conv_state.clone()
|
||||
# Exactly the call made by KimiK3DeltaAttention._forward.
|
||||
qf, kf, vf = fused_fn(
|
||||
mixed_qkv.transpose(0, 1),
|
||||
weight,
|
||||
None,
|
||||
state_f,
|
||||
qsl,
|
||||
proj,
|
||||
proj,
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial,
|
||||
activation="silu",
|
||||
block_m=block_m,
|
||||
metadata=None,
|
||||
preserve_input_dtype=True,
|
||||
)
|
||||
|
||||
fused = torch.cat([qf, kf, vf], dim=-1)
|
||||
assert fused.shape == mixed_qkv.shape and fused.dtype == mixed_qkv.dtype
|
||||
assert all(t.is_contiguous() for t in (qf, kf, vf))
|
||||
|
||||
tol = dict(atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(fused.float(), ref_out, **tol)
|
||||
torch.testing.assert_close(fused, torch.cat([q3, k3, v3], dim=-1), **tol)
|
||||
|
||||
# conv_state writeback: assigned lines updated, the others untouched.
|
||||
torch.testing.assert_close(state_f.float(), ref_state, **tol)
|
||||
torch.testing.assert_close(state_f, state_3, **tol)
|
||||
untouched = torch.ones(conv_state.shape[0], dtype=torch.bool, device="cuda")
|
||||
untouched[cache_indices.long()] = False
|
||||
assert untouched[NULL_BLOCK_ID]
|
||||
assert torch.equal(state_f[untouched], conv_state[untouched])
|
||||
@@ -1,441 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm latent-MoE tail must equal the replicated up-projection.
|
||||
|
||||
A wrong shard offset or a dropped accumulation still runs and still produces
|
||||
plausible text, so these pin the arithmetic rather than the behaviour.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
from torch.multiprocessing import spawn
|
||||
|
||||
from tests.utils import (
|
||||
ensure_current_vllm_config,
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
)
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.runner import moe_runner
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, ReplicatedLinear
|
||||
from vllm.models.kimi_k3.amd import latent_moe_runner
|
||||
from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner
|
||||
from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCmLatentMoERunner is only wired up on ROCm",
|
||||
)
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
EPS = 1e-5
|
||||
DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _build_transform(
|
||||
device: torch.device, *, row_sharded: bool = False
|
||||
) -> tuple[KimiRoutedOutputTransform, torch.Tensor]:
|
||||
norm = RMSNorm(LATENT_SIZE, eps=EPS).to(device=device, dtype=DTYPE)
|
||||
|
||||
torch.manual_seed(0)
|
||||
norm.weight.data.copy_(1 + 0.1 * torch.randn_like(norm.weight))
|
||||
full_weight = (
|
||||
torch.randn(HIDDEN_SIZE, LATENT_SIZE, device=device, dtype=DTYPE)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
|
||||
up_proj: ReplicatedLinear | ColumnParallelLinear
|
||||
if row_sharded:
|
||||
up_proj = ColumnParallelLinear(
|
||||
LATENT_SIZE,
|
||||
HIDDEN_SIZE,
|
||||
bias=False,
|
||||
gather_output=True,
|
||||
params_dtype=DTYPE,
|
||||
prefix="routed_expert_up_proj",
|
||||
).to(device=device)
|
||||
up_proj.weight_loader(up_proj.weight, full_weight)
|
||||
else:
|
||||
up_proj = ReplicatedLinear(
|
||||
LATENT_SIZE,
|
||||
HIDDEN_SIZE,
|
||||
bias=False,
|
||||
params_dtype=DTYPE,
|
||||
prefix="routed_expert_up_proj",
|
||||
).to(device=device)
|
||||
up_proj.weight.data.copy_(full_weight)
|
||||
return (
|
||||
KimiRoutedOutputTransform(norm, up_proj, row_sharded=row_sharded),
|
||||
full_weight,
|
||||
)
|
||||
|
||||
|
||||
def _tail_runner(
|
||||
transform: KimiRoutedOutputTransform, tp_world: int, use_ep: bool = False
|
||||
) -> ROCmLatentMoERunner:
|
||||
"""A runner carrying only what the tail reads, so no engine is needed.
|
||||
|
||||
``_maybe_reduce_final_output`` is left as the real base-class method, so the
|
||||
all-reduce that stitches the shards is the real collective.
|
||||
|
||||
``tp_world`` is the size of the TP process group, which is what the hidden
|
||||
dim is split by. Under expert parallelism the MoE config reports
|
||||
``tp_size=1``/``ep_size=tp_world`` for that same group, so ``use_ep`` pins
|
||||
that the final reduce still fires.
|
||||
"""
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
attrs = {
|
||||
"routed_output_transform": transform,
|
||||
"_up_proj_preshard": bool(getattr(transform, "row_sharded", False)),
|
||||
"_up_proj_shard_size": HIDDEN_SIZE // tp_world,
|
||||
"_logged_sharded_tail": False,
|
||||
"moe_config": SimpleNamespace(
|
||||
tp_size=1 if use_ep else tp_world,
|
||||
ep_size=tp_world if use_ep else 1,
|
||||
is_sequence_parallel=False,
|
||||
skip_final_all_reduce=False,
|
||||
),
|
||||
}
|
||||
for name, value in attrs.items():
|
||||
object.__setattr__(runner, name, value)
|
||||
return runner
|
||||
|
||||
|
||||
def _all_reduced(tensor: torch.Tensor, group) -> torch.Tensor:
|
||||
reduced = tensor.clone()
|
||||
dist.all_reduce(reduced, group=group)
|
||||
return reduced
|
||||
|
||||
|
||||
def _rank_partials(
|
||||
num_tokens: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routed = torch.randn(num_tokens, LATENT_SIZE, device=device, dtype=DTYPE)
|
||||
shared = torch.randn(num_tokens, HIDDEN_SIZE, device=device, dtype=DTYPE)
|
||||
return routed.mul_(0.01), shared
|
||||
|
||||
|
||||
def _check_matches_replicated(
|
||||
device: torch.device,
|
||||
tp_world: int,
|
||||
rank: int,
|
||||
use_ep: bool = False,
|
||||
row_sharded: bool = False,
|
||||
) -> None:
|
||||
transform, up_weight = _build_transform(device, row_sharded=row_sharded)
|
||||
runner = _tail_runner(transform, tp_world, use_ep)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 5)):
|
||||
torch.manual_seed(100 * iteration + rank + 1)
|
||||
routed_output, shared_output = _rank_partials(num_tokens, device)
|
||||
|
||||
expected = F.linear(
|
||||
F.rms_norm(
|
||||
_all_reduced(routed_output, group),
|
||||
(LATENT_SIZE,),
|
||||
transform.norm.weight,
|
||||
EPS,
|
||||
),
|
||||
up_weight,
|
||||
)
|
||||
expected.add_(_all_reduced(shared_output, group))
|
||||
|
||||
actual = runner._shard_up_proj_tail(routed_output, shared_output, None)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _check_matches_replicated_under_ep(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""The served config: the routed experts are expert-parallel, so the MoE
|
||||
config reports no tensor parallelism at all for a group that still splits
|
||||
the hidden dim ``tp_world`` ways."""
|
||||
_check_matches_replicated(device, tp_world, rank, use_ep=True)
|
||||
|
||||
|
||||
def _check_presharded_matches_replicated(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
_check_matches_replicated(device, tp_world, rank, row_sharded=True)
|
||||
|
||||
|
||||
def _check_unfused_tail_gathers_full_hidden(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
transform, up_weight = _build_transform(device, row_sharded=True)
|
||||
assert transform.up_proj.weight.shape[0] == HIDDEN_SIZE // tp_world
|
||||
|
||||
# all-gather reconstructs the full projection when all ranks feed the same
|
||||
# input, so use one seed for all ranks.
|
||||
torch.manual_seed(1234)
|
||||
latent = torch.randn(4, LATENT_SIZE, device=device, dtype=DTYPE).mul_(0.01)
|
||||
|
||||
torch.testing.assert_close(
|
||||
transform(latent),
|
||||
F.linear(transform.norm(latent), up_weight),
|
||||
atol=8e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
|
||||
|
||||
def _check_writes_only_its_own_shard(
|
||||
device: torch.device, tp_world: int, rank: int
|
||||
) -> None:
|
||||
"""Two ranks that swapped both weight rows and write offsets still sum to
|
||||
the right total, so inspect each slice before the final collective."""
|
||||
transform, up_weight = _build_transform(device)
|
||||
runner = _tail_runner(transform, tp_world)
|
||||
group = get_tp_group().device_group
|
||||
|
||||
torch.manual_seed(rank + 1)
|
||||
routed_output, shared_output = _rank_partials(8, device)
|
||||
before = shared_output.clone()
|
||||
latent = transform.norm(_all_reduced(routed_output, group))
|
||||
|
||||
captured: dict = {}
|
||||
real_reduce = runner._maybe_reduce_final_output
|
||||
|
||||
def _capture(states, trunc_size, output_is_reduced=None):
|
||||
captured["states"] = states.clone()
|
||||
captured["output_is_reduced"] = output_is_reduced
|
||||
return real_reduce(states, trunc_size, output_is_reduced)
|
||||
|
||||
object.__setattr__(runner, "_maybe_reduce_final_output", _capture)
|
||||
runner._shard_up_proj_tail(routed_output, shared_output, None)
|
||||
# Restoring the method breaks the runner -> _capture -> runner cycle, so the
|
||||
# captured device tensor is freed before the process group is torn down.
|
||||
object.__setattr__(runner, "_maybe_reduce_final_output", real_reduce)
|
||||
|
||||
assert captured["output_is_reduced"] is False
|
||||
|
||||
shard = HIDDEN_SIZE // tp_world
|
||||
start, end = rank * shard, (rank + 1) * shard
|
||||
local = captured["states"]
|
||||
projected = F.linear(latent, up_weight[start:end])
|
||||
|
||||
torch.testing.assert_close(
|
||||
local[:, start:end], before[:, start:end] + projected, atol=8e-2, rtol=3e-2
|
||||
)
|
||||
torch.testing.assert_close(local[:, :start], before[:, :start], atol=0, rtol=0)
|
||||
torch.testing.assert_close(local[:, end:], before[:, end:], atol=0, rtol=0)
|
||||
|
||||
|
||||
_CHECKS = {
|
||||
"matches_replicated": _check_matches_replicated,
|
||||
"matches_replicated_ep": _check_matches_replicated_under_ep,
|
||||
"presharded_matches_replicated": _check_presharded_matches_replicated,
|
||||
"unfused_gathers": _check_unfused_tail_gathers_full_hidden,
|
||||
"own_shard_only": _check_writes_only_its_own_shard,
|
||||
}
|
||||
|
||||
|
||||
def _worker(local_rank: int, world_size: int, port: str, check: str) -> None:
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
with ensure_current_vllm_config():
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
_CHECKS[check](device, world_size, local_rank)
|
||||
|
||||
|
||||
def _run_ranks(check: str, tp_size: int) -> None:
|
||||
spawn(
|
||||
_worker,
|
||||
args=(tp_size, str(get_open_port()), check),
|
||||
nprocs=tp_size,
|
||||
join=True,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 4)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_sharded_tail_tp8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_sharded_tail_tp8_ep8_matches_replicated_projection() -> None:
|
||||
_run_ranks("matches_replicated_ep", 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_presharded_tail_tp4_matches_replicated_projection() -> None:
|
||||
_run_ranks("presharded_matches_replicated", 4)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_presharded_unfused_tail_gathers_the_full_hidden_dim() -> None:
|
||||
_run_ranks("unfused_gathers", 4)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_sharded_tail_tp4_writes_only_its_own_shard() -> None:
|
||||
_run_ranks("own_shard_only", 4)
|
||||
|
||||
|
||||
def _runner(**attrs) -> ROCmLatentMoERunner:
|
||||
runner = object.__new__(ROCmLatentMoERunner)
|
||||
for name, value in attrs.items():
|
||||
object.__setattr__(runner, name, value)
|
||||
return runner
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"shardable,pre_reduced,expect_sharded",
|
||||
[
|
||||
(True, False, True),
|
||||
(True, True, False),
|
||||
(False, False, False),
|
||||
(False, True, False),
|
||||
],
|
||||
)
|
||||
def test_forward_shards_only_when_the_tail_is_valid(
|
||||
shardable: bool,
|
||||
pre_reduced: bool,
|
||||
expect_sharded: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The shard needs an un-reduced shared partial to accumulate into."""
|
||||
base_output, sharded_output = torch.zeros(1), torch.ones(1)
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "forward", lambda *a, **k: base_output)
|
||||
monkeypatch.setattr(
|
||||
ROCmLatentMoERunner, "_fused_forward", lambda *a, **k: sharded_output
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ROCmLatentMoERunner, "_fused_output_is_reduced", property(lambda _: pre_reduced)
|
||||
)
|
||||
runner = _runner(_tail_shardable=shardable)
|
||||
|
||||
result = runner.forward(torch.zeros(1), torch.zeros(1))
|
||||
|
||||
assert result is (sharded_output if expect_sharded else base_output)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def build_runner(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Construct through the real subclass ``__init__``, stubbing only the base.
|
||||
|
||||
``tp_world`` is the size of the TP process group and is varied independently
|
||||
of ``moe_config.tp_size``, because under expert parallelism those two are
|
||||
different numbers for the same eight ranks.
|
||||
"""
|
||||
world = {"size": 8}
|
||||
|
||||
def _base_init(
|
||||
self,
|
||||
*,
|
||||
tp_size: int = 8,
|
||||
ep_size: int = 1,
|
||||
hidden: int = HIDDEN_SIZE,
|
||||
is_sequence_parallel: bool = False,
|
||||
has_up_proj: bool = True,
|
||||
has_shared_experts: bool = True,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> None:
|
||||
self.moe_config = SimpleNamespace(
|
||||
tp_size=tp_size,
|
||||
ep_size=ep_size,
|
||||
is_sequence_parallel=is_sequence_parallel,
|
||||
)
|
||||
self.routed_output_transform = SimpleNamespace(
|
||||
norm=None,
|
||||
up_proj=SimpleNamespace(weight=torch.zeros(hidden, LATENT_SIZE))
|
||||
if has_up_proj
|
||||
else None,
|
||||
)
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self._shared_experts = object() if has_shared_experts else None
|
||||
|
||||
monkeypatch.setattr(moe_runner.MoERunner, "__init__", _base_init)
|
||||
monkeypatch.setattr(
|
||||
latent_moe_runner,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: world["size"],
|
||||
)
|
||||
|
||||
def _build(*, tp_world: int = 8, **kwargs) -> ROCmLatentMoERunner:
|
||||
world["size"] = tp_world
|
||||
return ROCmLatentMoERunner(**kwargs)
|
||||
|
||||
return _build
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config(build_runner) -> None:
|
||||
"""``--tensor-parallel-size 8``, MoE experts tensor-parallel."""
|
||||
runner = build_runner(tp_world=8, tp_size=8, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_under_the_kimi_k3_serving_config_with_expert_parallel(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""``--tensor-parallel-size 8 --enable-expert-parallel``, what is served.
|
||||
|
||||
EP reports ``moe_config.tp_size == 1`` because each rank owns whole experts,
|
||||
but the hidden dim is still split across all eight TP ranks, so the tail
|
||||
must shard exactly as it does without EP.
|
||||
"""
|
||||
runner = build_runner(tp_world=8, tp_size=1, ep_size=8)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 8
|
||||
|
||||
|
||||
def test_shards_by_the_process_group_not_the_dp_flattened_tp_size(
|
||||
build_runner,
|
||||
) -> None:
|
||||
"""Without EP, ``moe_config.tp_size`` is flattened across DP: TP=2 x DP=2
|
||||
reports 4 for a TP process group of 2.
|
||||
|
||||
Only those 2 ranks share a hidden dim and take part in the final
|
||||
all-reduce, so sharding by 4 would leave half of it with no routed
|
||||
contribution on either rank.
|
||||
"""
|
||||
runner = build_runner(tp_world=2, tp_size=4, ep_size=1)
|
||||
|
||||
assert runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == HIDDEN_SIZE // 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override",
|
||||
[
|
||||
pytest.param({"tp_world": 1, "tp_size": 1}, id="single-rank"),
|
||||
pytest.param({"tp_world": 1, "tp_size": 4}, id="dp-without-tp"),
|
||||
pytest.param({"hidden": HIDDEN_SIZE + 2}, id="hidden-not-divisible-by-tp"),
|
||||
pytest.param({"has_up_proj": False}, id="no-up-proj"),
|
||||
pytest.param({"has_shared_experts": False}, id="no-shared-partial"),
|
||||
pytest.param({"is_sequence_parallel": True}, id="sequence-parallel"),
|
||||
pytest.param({"routed_scaling_factor": 2.0}, id="routed-scaling-factor"),
|
||||
],
|
||||
)
|
||||
def test_falls_back_when_the_config_breaks_the_shard(
|
||||
build_runner, override: dict
|
||||
) -> None:
|
||||
"""Each of these makes the sharded tail wrong, not merely slower.
|
||||
|
||||
``dp-without-tp`` is DP=4 with no TP: the hidden dim is not split at all,
|
||||
so there is nothing to shard and no all-reduce to stitch shards back
|
||||
together, however large ``moe_config.tp_size`` claims to be.
|
||||
"""
|
||||
runner = build_runner(**override)
|
||||
|
||||
assert not runner._tail_shardable
|
||||
assert runner._up_proj_shard_size == 0
|
||||
@@ -1,119 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.kimi_k3.amd.linear import KimiDecoderLayer, KimiMLAAttention
|
||||
|
||||
|
||||
class _DirectMLA(KimiMLAAttention):
|
||||
def __init__(self, result: torch.Tensor) -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.result = result
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.result
|
||||
|
||||
|
||||
def _make_layer(self_attn: nn.Module) -> KimiDecoderLayer:
|
||||
layer = object.__new__(KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.self_attn = self_attn
|
||||
return layer
|
||||
|
||||
|
||||
def test_mla_self_attention_returns_projection_storage_directly() -> None:
|
||||
hidden_states = torch.randn(4, 8)
|
||||
projected = torch.randn_like(hidden_states)
|
||||
layer = _make_layer(_DirectMLA(projected))
|
||||
|
||||
output = layer._run_self_attn(torch.arange(4), hidden_states)
|
||||
|
||||
assert output is projected
|
||||
assert output.data_ptr() == projected.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantized", [False, True])
|
||||
@pytest.mark.parametrize("fused_decode", [False, True])
|
||||
def test_mla_forward_preserves_output_shape_with_fp8_input(
|
||||
monkeypatch, quantized: bool, fused_decode: bool
|
||||
) -> None:
|
||||
"""Both MLA paths accept producer-quantized input after decode fusion."""
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.mla import KimiK3MultiHeadLatentAttentionWrapper
|
||||
|
||||
hidden_states = torch.randn(4, 8, dtype=torch.bfloat16)
|
||||
if quantized:
|
||||
hidden_states = QuantizedActivation(
|
||||
hidden_states.to(kFp8DynamicTokenSym.dtype),
|
||||
torch.ones(4, 1),
|
||||
hidden_states.dtype,
|
||||
hidden_states.shape,
|
||||
kFp8DynamicTokenSym,
|
||||
)
|
||||
projected = torch.randn(4, 8, dtype=torch.bfloat16)
|
||||
paths = []
|
||||
|
||||
def input_projection(value):
|
||||
assert value is hidden_states
|
||||
return torch.zeros(4, 8, dtype=torch.bfloat16), None
|
||||
|
||||
def attention_output(path, output_shape):
|
||||
assert output_shape == projected.shape
|
||||
paths.append(path)
|
||||
return projected
|
||||
|
||||
attention = SimpleNamespace(
|
||||
q_lora_rank=2,
|
||||
kv_lora_rank=4,
|
||||
qk_rope_head_dim=2,
|
||||
qk_head_dim=4,
|
||||
num_heads=2,
|
||||
v_head_dim=4,
|
||||
fused_qkv_a_proj=input_projection,
|
||||
q_a_layernorm=nn.Identity(),
|
||||
_normalize_q_kv=lambda q, kv: (q, kv),
|
||||
q_b_proj=lambda q: (torch.zeros(4, 8, dtype=q.dtype), None),
|
||||
rotary_emb=None,
|
||||
indexer=None,
|
||||
dcp_q_replicate=False,
|
||||
_fused_qk_prep=fused_decode,
|
||||
_fused_decode=lambda q, kv, pe, pos, shape, *args: attention_output(
|
||||
"fused", shape
|
||||
),
|
||||
g_proj=None,
|
||||
o_proj=lambda value: (value, None),
|
||||
)
|
||||
if fused_decode:
|
||||
attention.mla_attn = SimpleNamespace(layer_name="test")
|
||||
monkeypatch.setattr(
|
||||
"vllm.models.kimi_k3.amd.mla.get_attention_context",
|
||||
lambda _: (
|
||||
SimpleNamespace(num_actual_tokens=4, num_decode_tokens=4),
|
||||
None,
|
||||
torch.zeros(1),
|
||||
torch.arange(4),
|
||||
),
|
||||
)
|
||||
else:
|
||||
attention.mla_attn = lambda q, kv, pe, **kwargs: attention_output(
|
||||
"regular", kwargs["output_shape"]
|
||||
)
|
||||
|
||||
output = KimiK3MultiHeadLatentAttentionWrapper.forward(
|
||||
attention, torch.arange(4), hidden_states
|
||||
)
|
||||
|
||||
assert output is projected
|
||||
assert paths == ["fused" if fused_decode else "regular"]
|
||||
@@ -1,302 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.common.mtp import fused_mtp_input
|
||||
from vllm.models.kimi_k3.nvidia.ops import attn_res
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
MAX_BLOCKS = 8
|
||||
EPS = 1e-5
|
||||
attn_res_module = importlib.import_module("vllm.models.kimi_k3.nvidia.ops.attn_res")
|
||||
|
||||
|
||||
def _on_rocm_below_10() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import get_rocm_version
|
||||
|
||||
return (get_rocm_version() or (0,)) < (10,)
|
||||
|
||||
|
||||
# The Triton bundled with ROCm < 10 (3.7.x) crashes in the AMD
|
||||
# CanonicalizePointers pass on the kernel's tl.where over pointer tensors. Only
|
||||
# the num_blocks > 0 loop contains it. ROCm serves AttnRes from
|
||||
# vllm/models/kimi_k3/amd/ops/attn_res.py instead, so nothing real is lost.
|
||||
_OLD_ROCM = _on_rocm_below_10()
|
||||
|
||||
|
||||
def _skip_on_old_rocm(num_blocks: int) -> None:
|
||||
if _OLD_ROCM and num_blocks > 0:
|
||||
pytest.skip("Triton on ROCm < 10 cannot compile this kernel's pointer select")
|
||||
|
||||
|
||||
def _randn_with_row_padding(*shape: int, padding: int = 0) -> torch.Tensor:
|
||||
storage = torch.randn(
|
||||
*shape[:-1],
|
||||
shape[-1] + padding,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
return storage[..., : shape[-1]]
|
||||
|
||||
|
||||
def _reference(
|
||||
prefix: torch.Tensor,
|
||||
delta: torch.Tensor | None,
|
||||
blocks: torch.Tensor,
|
||||
norm_weight: torch.Tensor,
|
||||
qk_weight: torch.Tensor,
|
||||
output_norm_weight: torch.Tensor | None,
|
||||
num_blocks: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if delta is not None:
|
||||
prefix = prefix + delta
|
||||
values = torch.cat((blocks[:, :num_blocks], prefix.unsqueeze(1)), dim=1)
|
||||
keys = F.rms_norm(values, (HIDDEN_SIZE,), norm_weight, EPS)
|
||||
probs = (keys @ qk_weight).softmax(dim=-1)
|
||||
output = torch.matmul(probs.unsqueeze(1), values).squeeze(1)
|
||||
if output_norm_weight is not None:
|
||||
output = F.rms_norm(output, (HIDDEN_SIZE,), output_norm_weight, EPS)
|
||||
return output, prefix
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"num_tokens",
|
||||
"num_blocks",
|
||||
"row_padding",
|
||||
"write_block",
|
||||
"has_delta",
|
||||
"backend",
|
||||
),
|
||||
[
|
||||
pytest.param(1, 0, 0, True, False, "triton", id="triton-empty"),
|
||||
pytest.param(1, 0, 0, True, True, "triton", id="triton-empty-add"),
|
||||
pytest.param(17, 5, 7, True, False, "triton", id="triton-write"),
|
||||
pytest.param(17, 5, 7, True, True, "triton", id="triton-write-add"),
|
||||
pytest.param(3, 8, 0, False, False, "triton", id="triton-full"),
|
||||
pytest.param(3, 8, 0, False, True, "triton", id="triton-full-add"),
|
||||
pytest.param(1, 4, 0, False, True, "nvidia", id="nvidia-single-token"),
|
||||
pytest.param(320, 1, 0, False, True, "nvidia", id="nvidia-1"),
|
||||
pytest.param(320, 4, 0, False, True, "nvidia", id="nvidia-4"),
|
||||
pytest.param(320, 8, 0, False, True, "nvidia", id="nvidia-8"),
|
||||
],
|
||||
)
|
||||
def test_attn_res(
|
||||
num_tokens: int,
|
||||
num_blocks: int,
|
||||
row_padding: int,
|
||||
write_block: bool,
|
||||
has_delta: bool,
|
||||
backend: str,
|
||||
):
|
||||
if backend == "nvidia" and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("NVIDIA AttnRes requires the SM100 family")
|
||||
_skip_on_old_rocm(num_blocks)
|
||||
|
||||
prefix = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
|
||||
delta = (
|
||||
_randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=row_padding)
|
||||
if has_delta
|
||||
else None
|
||||
)
|
||||
blocks = _randn_with_row_padding(
|
||||
num_tokens, MAX_BLOCKS, HIDDEN_SIZE, padding=row_padding
|
||||
)
|
||||
norm_weight = 1 + 0.1 * torch.randn(
|
||||
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
output_norm_weight = 1 + 0.1 * torch.randn(
|
||||
HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
original_blocks = blocks.clone()
|
||||
expected, expected_prefix = _reference(
|
||||
prefix.clone(),
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
)
|
||||
block_write_idx = num_blocks if write_block else -1
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
block_write_idx,
|
||||
EPS,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
torch.testing.assert_close(prefix, expected_prefix, atol=0, rtol=0)
|
||||
if write_block:
|
||||
original_blocks[:, block_write_idx].copy_(expected_prefix)
|
||||
torch.testing.assert_close(blocks, original_blocks, atol=0, rtol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_blocks", range(MAX_BLOCKS + 1))
|
||||
def test_attn_res_block_counts(num_blocks: int):
|
||||
_skip_on_old_rocm(num_blocks)
|
||||
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
blocks = torch.randn(
|
||||
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
output_norm_weight = torch.ones_like(norm_weight)
|
||||
expected, _ = _reference(
|
||||
prefix.clone(),
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
)
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
None,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
-1,
|
||||
EPS,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def test_attn_res_without_output_norm():
|
||||
_skip_on_old_rocm(MAX_BLOCKS)
|
||||
prefix = torch.randn(7, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
delta = torch.randn_like(prefix)
|
||||
blocks = torch.randn(
|
||||
7, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
norm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
qk_weight = (
|
||||
torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) / HIDDEN_SIZE**0.5
|
||||
)
|
||||
expected, _ = _reference(
|
||||
prefix.clone(), delta, blocks, norm_weight, qk_weight, None, MAX_BLOCKS
|
||||
)
|
||||
|
||||
actual = attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
None,
|
||||
MAX_BLOCKS,
|
||||
-1,
|
||||
EPS,
|
||||
0.0,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_blocks", "has_delta", "has_output_norm", "write_block"),
|
||||
[
|
||||
(0, False, True, True),
|
||||
(4, True, True, True),
|
||||
(4, False, True, False),
|
||||
(8, True, False, False),
|
||||
],
|
||||
)
|
||||
def test_sm100_variants_do_not_fall_back_to_triton(
|
||||
monkeypatch,
|
||||
num_blocks: int,
|
||||
has_delta: bool,
|
||||
has_output_norm: bool,
|
||||
write_block: bool,
|
||||
):
|
||||
if not current_platform.is_device_capability_family(100) or not hasattr(
|
||||
torch.ops._C, "kimi_k3_attn_res"
|
||||
):
|
||||
pytest.skip("native AttnRes requires an SM100 build")
|
||||
|
||||
class FailingKernel:
|
||||
def __getitem__(self, grid):
|
||||
raise AssertionError(f"unexpected Triton launch with grid {grid}")
|
||||
|
||||
monkeypatch.setattr(attn_res_module, "_attn_res_kernel", FailingKernel())
|
||||
prefix = torch.randn(1, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
delta = torch.zeros_like(prefix) if has_delta else None
|
||||
blocks = torch.randn(
|
||||
1, MAX_BLOCKS, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
norm_weight = torch.ones(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
qk_weight = torch.randn_like(norm_weight) / HIDDEN_SIZE**0.5
|
||||
output_norm_weight = torch.ones_like(norm_weight) if has_output_norm else None
|
||||
|
||||
attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
blocks,
|
||||
norm_weight,
|
||||
qk_weight,
|
||||
output_norm_weight,
|
||||
num_blocks,
|
||||
num_blocks if write_block else -1,
|
||||
EPS,
|
||||
EPS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 17])
|
||||
def test_fused_mtp_input(num_tokens: int):
|
||||
positions = torch.arange(num_tokens, device="cuda")
|
||||
inputs_embeds = _randn_with_row_padding(num_tokens, HIDDEN_SIZE, padding=7)
|
||||
previous_hidden_states = _randn_with_row_padding(
|
||||
num_tokens, HIDDEN_SIZE, padding=11
|
||||
)
|
||||
enorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
hnorm_weight = torch.randn(HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
masked_inputs_embeds = torch.where(positions.unsqueeze(-1) == 0, 0, inputs_embeds)
|
||||
expected = torch.cat(
|
||||
(
|
||||
F.rms_norm(masked_inputs_embeds, (HIDDEN_SIZE,), enorm_weight, EPS),
|
||||
F.rms_norm(previous_hidden_states, (HIDDEN_SIZE,), hnorm_weight, EPS),
|
||||
),
|
||||
dim=-1,
|
||||
)
|
||||
actual = fused_mtp_input(
|
||||
positions,
|
||||
inputs_embeds,
|
||||
previous_hidden_states,
|
||||
enorm_weight,
|
||||
hnorm_weight,
|
||||
EPS,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
assert actual.shape == (num_tokens, 2 * HIDDEN_SIZE)
|
||||
assert actual.is_contiguous()
|
||||
@@ -1,195 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Which value the DFlash drafter is fed under AttnRes.
|
||||
|
||||
`_capture_aux_hidden_stream` picks the weights it mixes against from one of
|
||||
three places depending on where the tapped layer sits, and returns the plain
|
||||
running prefix when the feature is off. The mixture itself is the kernel's
|
||||
job and is covered by ``test_attn_res.py``; what is asserted here is the
|
||||
selection, which is the part that can silently feed the drafter the wrong
|
||||
tensor.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia import model as k3_model
|
||||
|
||||
END_LAYER = 4
|
||||
|
||||
|
||||
def _weights(tag: float) -> SimpleNamespace:
|
||||
"""A norm/projection pair that is identifiable by value."""
|
||||
return SimpleNamespace(
|
||||
weight=torch.full((2,), tag),
|
||||
variance_epsilon=tag,
|
||||
)
|
||||
|
||||
|
||||
def _stub_model(*, enabled: bool, use_attn_res: bool = True) -> SimpleNamespace:
|
||||
"""A stand-in carrying only what the tap reads.
|
||||
|
||||
Constructing the real model needs a distributed init and weights, and none
|
||||
of it participates in the selection under test.
|
||||
"""
|
||||
consumers = []
|
||||
for i in range(END_LAYER):
|
||||
consumers.append(
|
||||
SimpleNamespace(
|
||||
self_attention_res_norm=_weights(float(i)),
|
||||
self_attention_res_proj=SimpleNamespace(
|
||||
weight=torch.full((1, 2), float(i))
|
||||
),
|
||||
prev_valid_blocks=i,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
_aux_attn_res_stream=enabled,
|
||||
use_attn_res=use_attn_res,
|
||||
end_layer=END_LAYER,
|
||||
layers=consumers,
|
||||
output_attn_res_norm=_weights(99.0),
|
||||
output_attn_res_proj=SimpleNamespace(weight=torch.full((1, 2), 99.0)),
|
||||
num_attn_res_blocks=99,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorder(monkeypatch):
|
||||
"""Replace the kernel so the call it would have made is inspectable."""
|
||||
calls = []
|
||||
|
||||
def _fake_attn_res(
|
||||
prefix,
|
||||
delta,
|
||||
block_residual,
|
||||
norm_weight,
|
||||
proj_weight,
|
||||
output_norm_weight,
|
||||
**kwargs,
|
||||
):
|
||||
calls.append(
|
||||
SimpleNamespace(
|
||||
prefix=prefix,
|
||||
delta=delta,
|
||||
block_residual=block_residual,
|
||||
norm_weight=norm_weight,
|
||||
proj_weight=proj_weight,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
)
|
||||
return torch.full_like(prefix, -1.0)
|
||||
|
||||
monkeypatch.setattr(k3_model, "attn_res", _fake_attn_res)
|
||||
return calls
|
||||
|
||||
|
||||
def _set_last_rank(monkeypatch, is_last: bool):
|
||||
monkeypatch.setattr(
|
||||
k3_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_last_rank=is_last),
|
||||
)
|
||||
|
||||
|
||||
def _call(stub, layer_idx, prefix_sum, pending_mlp_out, block_residual):
|
||||
return k3_model.KimiLinearModel._capture_aux_hidden_stream(
|
||||
stub, layer_idx, prefix_sum, pending_mlp_out, block_residual
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,use_attn_res", [(False, True), (True, False), (False, False)]
|
||||
)
|
||||
def test_disabled_reproduces_the_plain_residual_sum(
|
||||
recorder, monkeypatch, enabled, use_attn_res
|
||||
):
|
||||
"""Off, the tap must be exactly the sum it replaced.
|
||||
|
||||
Both conditions matter. `use_attn_res` is what constructs the norm and
|
||||
projection weights, so without it the lookups below would raise rather
|
||||
than fall back.
|
||||
"""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
prefix_sum = torch.tensor([1.0, 2.0])
|
||||
pending = torch.tensor([0.5, 0.25])
|
||||
|
||||
got = _call(
|
||||
_stub_model(enabled=enabled, use_attn_res=use_attn_res),
|
||||
0,
|
||||
prefix_sum,
|
||||
pending,
|
||||
torch.zeros(2),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, prefix_sum + pending)
|
||||
assert not recorder, "the kernel must not run when the tap is off"
|
||||
|
||||
|
||||
def test_taps_the_consumer_layer_when_one_follows(recorder, monkeypatch):
|
||||
"""The value the next layer reads is the mixture against *its* weights,
|
||||
so the tap has to reach forward rather than use the current layer's."""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
|
||||
_call(_stub_model(enabled=True), 1, torch.zeros(2), None, torch.zeros(2))
|
||||
|
||||
assert len(recorder) == 1
|
||||
call = recorder[0]
|
||||
# Layer 2's weights, not layer 1's.
|
||||
torch.testing.assert_close(call.norm_weight, torch.full((2,), 2.0))
|
||||
assert call.kwargs["num_blocks"] == 2
|
||||
|
||||
|
||||
def test_last_layer_on_the_final_rank_uses_the_output_aggregation(
|
||||
recorder, monkeypatch
|
||||
):
|
||||
"""Nothing downstream but the model's own output-side mixture."""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
|
||||
_call(
|
||||
_stub_model(enabled=True), END_LAYER - 1, torch.zeros(2), None, torch.zeros(2)
|
||||
)
|
||||
|
||||
assert len(recorder) == 1
|
||||
torch.testing.assert_close(recorder[0].norm_weight, torch.full((2,), 99.0))
|
||||
assert recorder[0].kwargs["num_blocks"] == 99
|
||||
|
||||
|
||||
def test_aux_layer_at_non_final_pp_boundary_is_rejected(monkeypatch):
|
||||
model = k3_model.KimiLinearModel.__new__(k3_model.KimiLinearModel)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_attn_res = True
|
||||
model.end_layer = 72
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.parallel_state.model_parallel_is_initialized", lambda: False
|
||||
)
|
||||
_set_last_rank(monkeypatch, False)
|
||||
monkeypatch.setattr(
|
||||
k3_model.KimiLinearModel,
|
||||
"_aux_attn_res_stream",
|
||||
property(lambda self: True),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Auxiliary layer 72"):
|
||||
model._set_aux_hidden_state_layers((3, 24, 48, 72, 90))
|
||||
|
||||
|
||||
def test_pending_mlp_output_is_folded_in_rather_than_passed_as_delta(
|
||||
recorder, monkeypatch
|
||||
):
|
||||
"""The kernel writes an applied delta back into the prefix in place, which
|
||||
would double-add it into the live residual stream, so the pending output
|
||||
has to arrive already summed into the prefix with `delta` left None."""
|
||||
_set_last_rank(monkeypatch, True)
|
||||
prefix_sum = torch.tensor([1.0, 2.0])
|
||||
pending = torch.tensor([0.5, 0.25])
|
||||
|
||||
_call(_stub_model(enabled=True), 0, prefix_sum, pending, torch.zeros(2))
|
||||
|
||||
assert len(recorder) == 1
|
||||
assert recorder[0].delta is None
|
||||
torch.testing.assert_close(recorder[0].prefix, prefix_sum + pending)
|
||||
# And the caller's tensor is not mutated on the way.
|
||||
torch.testing.assert_close(prefix_sum, torch.tensor([1.0, 2.0]))
|
||||
@@ -1,202 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.interfaces import supports_eagle3
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia.model import (
|
||||
KimiK3ForConditionalGeneration,
|
||||
KimiLinearForCausalLM,
|
||||
KimiLinearModel,
|
||||
)
|
||||
|
||||
|
||||
def _make_kimi_linear_model() -> KimiLinearModel:
|
||||
model = object.__new__(KimiLinearModel)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (2,))
|
||||
object.__setattr__(model, "use_sequence_parallel", False)
|
||||
object.__setattr__(model, "use_attn_res", False)
|
||||
return model
|
||||
|
||||
|
||||
def test_kimi_k3_advertises_eagle3_support():
|
||||
assert supports_eagle3(KimiK3ForConditionalGeneration)
|
||||
|
||||
|
||||
def test_kimi_linear_advertises_eagle3_support():
|
||||
# The text-only architecture serves the same inner KimiLinearModel, which
|
||||
# already carries the EagleModelMixin tap machinery - only the interface
|
||||
# declaration was missing, so EAGLE3-family speculative decoding (dspark)
|
||||
# was rejected at startup with "Model does not support EAGLE3 interface".
|
||||
assert supports_eagle3(KimiLinearForCausalLM)
|
||||
|
||||
|
||||
def test_kimi_k3_uses_shared_eagle3_layer_configuration():
|
||||
target = object.__new__(KimiK3ForConditionalGeneration)
|
||||
torch.nn.Module.__init__(target)
|
||||
model = _make_kimi_linear_model()
|
||||
object.__setattr__(model, "layers", [None] * 93)
|
||||
language_model = SimpleNamespace(
|
||||
embed_input_ids=lambda _: None,
|
||||
forward=lambda input_ids, positions: None,
|
||||
model=model,
|
||||
)
|
||||
object.__setattr__(target, "language_model", language_model)
|
||||
object.__setattr__(target, "_language_model_names", ["language_model"])
|
||||
|
||||
target.set_aux_hidden_state_layers((2, 46, 90))
|
||||
|
||||
assert model.aux_hidden_state_layers == (2, 46, 90)
|
||||
assert target.get_eagle3_default_aux_hidden_state_layers() == (
|
||||
2,
|
||||
46,
|
||||
90,
|
||||
)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_standard_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
layer_residual = torch.tensor([[5.0, 6.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, None, layer_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", False)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
expected_layer_output = layer_hidden_states + layer_residual
|
||||
torch.testing.assert_close(output, expected_layer_output)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], expected_layer_output)
|
||||
|
||||
|
||||
def test_kimi_linear_forward_extracts_attn_res_aux_hidden_states(monkeypatch):
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
prefix_sum = torch.tensor([[5.0, 6.0]])
|
||||
block_residual = torch.tensor([[[7.0, 8.0]]])
|
||||
final_hidden_states = torch.tensor([[9.0, 10.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (0, 1))
|
||||
object.__setattr__(model, "use_attn_res", True)
|
||||
object.__setattr__(model, "num_attn_res_blocks", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_norm",
|
||||
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
|
||||
)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_proj",
|
||||
SimpleNamespace(weight=torch.ones(1, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
final_attn_res = Mock(return_value=final_hidden_states)
|
||||
monkeypatch.setattr(kimi_model, "attn_res", final_attn_res)
|
||||
|
||||
output, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, final_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[0], initial_hidden_states)
|
||||
torch.testing.assert_close(aux_hidden_states[1], prefix_sum + layer_hidden_states)
|
||||
assert final_attn_res.call_args.args[2] is block_residual
|
||||
|
||||
|
||||
def test_attn_res_stream_capture_receives_the_layer_outputs_in_order(monkeypatch):
|
||||
"""Pin the argument mapping at the call site.
|
||||
|
||||
The capture helper's own tests invoke it directly by keyword, so they
|
||||
cannot catch a swap where `forward` hands it the residual as the pending
|
||||
MLP output. Both are tensors of the same shape, so a swap is silent: it
|
||||
feeds the drafter a wrong but well-formed tensor.
|
||||
"""
|
||||
model = _make_kimi_linear_model()
|
||||
initial_hidden_states = torch.tensor([[1.0, 2.0]])
|
||||
layer_hidden_states = torch.tensor([[3.0, 4.0]])
|
||||
prefix_sum = torch.tensor([[5.0, 6.0]])
|
||||
block_residual = torch.tensor([[[7.0, 8.0]]])
|
||||
captured = torch.tensor([[11.0, 12.0]])
|
||||
|
||||
object.__setattr__(model, "start_layer", 0)
|
||||
object.__setattr__(model, "end_layer", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"layers",
|
||||
[Mock(return_value=(layer_hidden_states, prefix_sum, block_residual))],
|
||||
)
|
||||
object.__setattr__(model, "aux_hidden_state_layers", (1,))
|
||||
object.__setattr__(model, "use_attn_res", True)
|
||||
object.__setattr__(model, "num_attn_res_blocks", 1)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_norm",
|
||||
SimpleNamespace(weight=torch.ones(2), variance_epsilon=1e-5),
|
||||
)
|
||||
object.__setattr__(
|
||||
model,
|
||||
"output_attn_res_proj",
|
||||
SimpleNamespace(weight=torch.ones(1, 2)),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"get_pp_group",
|
||||
lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True),
|
||||
)
|
||||
monkeypatch.setattr(kimi_model, "attn_res", Mock(return_value=torch.zeros(1, 2)))
|
||||
monkeypatch.setenv("VLLM_KIMI_K3_AUX_ATTN_RES_STREAM", "1")
|
||||
|
||||
capture = Mock(return_value=captured)
|
||||
monkeypatch.setattr(KimiLinearModel, "_capture_aux_hidden_stream", capture)
|
||||
|
||||
_, aux_hidden_states = model.forward(
|
||||
input_ids=None,
|
||||
positions=torch.tensor([0]),
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds=initial_hidden_states,
|
||||
)
|
||||
|
||||
layer_idx, got_prefix, got_pending, got_residual = capture.call_args.args
|
||||
assert layer_idx == 0
|
||||
assert got_prefix is prefix_sum
|
||||
assert got_pending is layer_hidden_states
|
||||
assert got_residual is block_residual
|
||||
torch.testing.assert_close(aux_hidden_states[0], captured)
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
_moe_weight_override_is_int4,
|
||||
_use_k3_situ_int4_gfx942,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeRocmModule(ModuleType):
|
||||
on_gfx942: Callable[[], bool]
|
||||
|
||||
|
||||
class _FakeAiterOpsModule(ModuleType):
|
||||
rocm_aiter_ops: SimpleNamespace
|
||||
|
||||
|
||||
def _vllm_config(moe_weight: str | None):
|
||||
quantization_config = (
|
||||
None
|
||||
if moe_weight is None
|
||||
else QuantizationConfigArgs(moe={"weight": moe_weight})
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(quantization_config=quantization_config)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("moe_weight", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
("mxfp4", False),
|
||||
("int4_per_group_32", True),
|
||||
],
|
||||
)
|
||||
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.get_current_vllm_config",
|
||||
lambda: _vllm_config(moe_weight),
|
||||
)
|
||||
assert _moe_weight_override_is_int4() is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("int4_requested", [False, True])
|
||||
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
|
||||
moe = SimpleNamespace(
|
||||
activation=MoEActivation.SITU,
|
||||
activation_situ_linear_beta=25.0,
|
||||
)
|
||||
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
|
||||
fake_rocm.on_gfx942 = lambda: True
|
||||
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
|
||||
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"vllm.platforms.rocm": fake_rocm,
|
||||
"vllm._aiter_ops": fake_aiter_ops,
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.mxfp4."
|
||||
"_moe_weight_override_is_int4",
|
||||
return_value=int4_requested,
|
||||
),
|
||||
):
|
||||
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
|
||||
|
||||
|
||||
def test_int4_per_group_32_is_not_an_online_moe_method():
|
||||
# Load-time gfx942 requant. Must not match the online MoE table.
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
|
||||
)
|
||||
assert (
|
||||
config.resolve_quant_method_cls(RoutedExperts, "model.layers.0.mlp.experts")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_online_moe_weight_still_raises():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
|
||||
)
|
||||
with pytest.raises(ValueError, match="online quantization"):
|
||||
config.resolve_quant_method_cls(RoutedExperts, "model.layers.0.mlp.experts")
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,482 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import (
|
||||
init_test_distributed_environment,
|
||||
multi_gpu_test,
|
||||
multi_process_parallel,
|
||||
)
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner
|
||||
from vllm.model_executor.warmup.cutedsl_warmup import cutedsl_warmup
|
||||
from vllm.models.kimi_k3.nvidia import latent_moe_runner
|
||||
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import KimiK3LatentMoETailOp
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HIDDEN_SIZE = 7168
|
||||
LATENT_SIZE = 3584
|
||||
EPS = 0.1
|
||||
TOP_K = 8
|
||||
|
||||
|
||||
def test_deferred_finalize_enabled_before_moe_kernel_setup(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class FakeMoEConfig:
|
||||
tp_size = 8
|
||||
dp_size = 1
|
||||
ep_size = 1
|
||||
pcp_size = 1
|
||||
is_sequence_parallel = False
|
||||
hidden_dim = LATENT_SIZE
|
||||
hidden_dim_unpadded = LATENT_SIZE
|
||||
experts_per_token = 16
|
||||
_defer_moe_finalize = False
|
||||
defer_moe_finalize_max_num_tokens = -1
|
||||
defer_moe_finalize = FusedMoEConfig.defer_moe_finalize
|
||||
limit_deferred_moe_finalize = FusedMoEConfig.limit_deferred_moe_finalize
|
||||
use_deferred_moe_finalize = FusedMoEConfig.use_deferred_moe_finalize
|
||||
|
||||
moe_config = FakeMoEConfig()
|
||||
quant_method = SimpleNamespace(
|
||||
experts_cls=TrtLlmMxfp4ExpertsMonolithic,
|
||||
moe_kernel=None,
|
||||
)
|
||||
norm_weight = torch.empty(LATENT_SIZE, dtype=torch.bfloat16)
|
||||
transform = SimpleNamespace(
|
||||
norm=SimpleNamespace(weight=norm_weight, variance_epsilon=EPS),
|
||||
up_proj=SimpleNamespace(
|
||||
weight=SimpleNamespace(shape=(HIDDEN_SIZE, LATENT_SIZE))
|
||||
),
|
||||
)
|
||||
|
||||
def fake_runner_init(runner, *args, **kwargs) -> None:
|
||||
runner.moe_config = moe_config
|
||||
runner.routed_experts = SimpleNamespace(quant_method=quant_method)
|
||||
runner._shared_experts = object()
|
||||
runner.routed_output_transform = transform
|
||||
|
||||
initialized_with: dict[str, object] = {}
|
||||
tail_op = SimpleNamespace(contract=SimpleNamespace(max_num_tokens=128))
|
||||
|
||||
def fake_tail_initialize(**kwargs):
|
||||
initialized_with.update(kwargs)
|
||||
return tail_op
|
||||
|
||||
monkeypatch.setattr(MoERunner, "__init__", fake_runner_init)
|
||||
monkeypatch.setattr(latent_moe_runner.torch.cuda, "Event", lambda: object())
|
||||
monkeypatch.setattr(
|
||||
latent_moe_runner,
|
||||
"current_platform",
|
||||
SimpleNamespace(
|
||||
is_cuda=lambda: True,
|
||||
is_device_capability_family=lambda capability: capability == 100,
|
||||
current_device=lambda: torch.device("cuda"),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
latent_moe_runner,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(
|
||||
parallel_config=SimpleNamespace(use_ubatching=False),
|
||||
model_config=SimpleNamespace(enable_sleep_mode=False),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(KimiK3LatentMoETailOp, "initialize", fake_tail_initialize)
|
||||
|
||||
latent_moe_runner.LatentMoERunner()
|
||||
|
||||
assert moe_config.use_deferred_moe_finalize
|
||||
assert moe_config.defer_moe_finalize_max_num_tokens == 128
|
||||
assert initialized_with["experts_per_token"] == 16
|
||||
|
||||
|
||||
def _make_deferred_routed_output(
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
|
||||
num_routes = num_tokens * TOP_K
|
||||
num_permuted_rows = num_routes + 7
|
||||
expanded_output = torch.randn(
|
||||
num_routes,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).mul_(0.01)
|
||||
expert_weights = torch.rand(
|
||||
num_tokens,
|
||||
TOP_K,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
|
||||
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
|
||||
gemm2_permuted = torch.empty(
|
||||
num_permuted_rows,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
gemm2_permuted[expanded_idx] = expanded_output
|
||||
|
||||
finalized = torch.zeros(
|
||||
num_tokens,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
expanded_output = expanded_output.view(num_tokens, TOP_K, LATENT_SIZE)
|
||||
for slot in range(TOP_K):
|
||||
finalized.add_(expanded_output[:, slot] * expert_weights[:, slot, None])
|
||||
|
||||
return (
|
||||
UnfinalizedMoEOutput(
|
||||
gemm2_permuted=gemm2_permuted,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32).view(
|
||||
num_tokens, TOP_K
|
||||
),
|
||||
),
|
||||
finalized,
|
||||
)
|
||||
|
||||
|
||||
def _make_bf16_top16_deferred_output(
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
*,
|
||||
drop_last_route: bool = False,
|
||||
) -> tuple[UnfinalizedMoEOutput, torch.Tensor]:
|
||||
top_k = 16
|
||||
num_routes = num_tokens * top_k
|
||||
num_permuted_rows = num_routes + 7
|
||||
expanded_output = torch.randn(
|
||||
num_routes,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).mul_(0.01)
|
||||
expert_weights = torch.rand(
|
||||
num_tokens,
|
||||
top_k,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
expert_weights.div_(expert_weights.sum(dim=-1, keepdim=True))
|
||||
expanded_idx = torch.randperm(num_permuted_rows, device=device)[:num_routes]
|
||||
gemm2_permuted = torch.empty(
|
||||
num_permuted_rows,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
gemm2_permuted[expanded_idx] = expanded_output
|
||||
|
||||
expanded_output = expanded_output.view(num_tokens, top_k, LATENT_SIZE)
|
||||
expanded_idx = expanded_idx.view(num_tokens, top_k)
|
||||
if drop_last_route:
|
||||
expanded_output[:, -1].zero_()
|
||||
expanded_idx[:, -1] = -1
|
||||
finalized = (
|
||||
(expanded_output.float() * expert_weights[:, :, None].float())
|
||||
.sum(1)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
return (
|
||||
UnfinalizedMoEOutput(
|
||||
gemm2_permuted=gemm2_permuted,
|
||||
expert_weights=expert_weights,
|
||||
expanded_idx_to_permuted_idx=expanded_idx.to(torch.int32),
|
||||
),
|
||||
finalized,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _test_latent_moe_tail_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
rms_weight = 1 + 0.1 * torch.randn(
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
up_weight = (
|
||||
torch.randn(
|
||||
HIDDEN_SIZE,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
|
||||
group = get_tp_group().device_group
|
||||
op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
)
|
||||
cutedsl_warmup()
|
||||
|
||||
# M=65 crosses two 32-CTA token waves and reuses the first DSM slot.
|
||||
for iteration, num_tokens in enumerate((1, 5, 8, 16, 33, 65, 5)):
|
||||
torch.manual_seed(100 * iteration + rank + 1)
|
||||
routed_output = torch.randn(
|
||||
num_tokens,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).mul_(0.01)
|
||||
shared_output = torch.randn(
|
||||
num_tokens,
|
||||
HIDDEN_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
routed_reference = routed_output.clone()
|
||||
shared_reference = shared_output.clone()
|
||||
dist.all_reduce(routed_reference, group=group)
|
||||
dist.all_reduce(shared_reference, group=group)
|
||||
expected = F.linear(
|
||||
F.rms_norm(
|
||||
routed_reference,
|
||||
(LATENT_SIZE,),
|
||||
rms_weight,
|
||||
EPS,
|
||||
),
|
||||
up_weight,
|
||||
)
|
||||
expected.add_(shared_reference)
|
||||
|
||||
actual = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_output = op(
|
||||
routed_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(graph_output, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
|
||||
def _run_latent_moe_tail_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("K3 latent-MoE tail fusion requires SM100")
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
1,
|
||||
_test_latent_moe_tail_worker,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def _test_deferred_finalize_parity_worker(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
rank: int,
|
||||
distributed_init_port: str,
|
||||
) -> None:
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
)
|
||||
|
||||
torch.manual_seed(1000 + rank)
|
||||
rms_weight = 1 + 0.1 * torch.randn(
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
up_weight = (
|
||||
torch.randn(
|
||||
HIDDEN_SIZE,
|
||||
LATENT_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
/ LATENT_SIZE**0.5
|
||||
)
|
||||
finalized_op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
)
|
||||
deferred_op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
experts_per_token=TOP_K,
|
||||
)
|
||||
bf16_deferred_op = KimiK3LatentMoETailOp.initialize(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
latent_size=LATENT_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
rms_eps=EPS,
|
||||
experts_per_token=16,
|
||||
)
|
||||
cutedsl_warmup()
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 5, 16, 33, 65)):
|
||||
torch.manual_seed(2000 + 100 * iteration + rank)
|
||||
deferred_output, finalized_output = _make_deferred_routed_output(
|
||||
num_tokens,
|
||||
device,
|
||||
)
|
||||
shared_output = torch.randn(
|
||||
num_tokens,
|
||||
HIDDEN_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
expected = finalized_op(
|
||||
finalized_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
actual = deferred_op(
|
||||
deferred_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
|
||||
for iteration, num_tokens in enumerate((1, 2, 4, 5, 16)):
|
||||
torch.manual_seed(3000 + 100 * iteration + rank)
|
||||
deferred_output, finalized_output = _make_bf16_top16_deferred_output(
|
||||
num_tokens,
|
||||
device,
|
||||
drop_last_route=num_tokens == 5,
|
||||
)
|
||||
shared_output = torch.randn(
|
||||
num_tokens,
|
||||
HIDDEN_SIZE,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
expected = finalized_op(
|
||||
finalized_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
actual = bf16_deferred_op(
|
||||
deferred_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=8e-2, rtol=3e-2)
|
||||
if num_tokens == 4:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_actual = bf16_deferred_op(
|
||||
deferred_output,
|
||||
shared_output,
|
||||
rms_weight,
|
||||
up_weight,
|
||||
)
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
graph_actual,
|
||||
expected,
|
||||
atol=8e-2,
|
||||
rtol=3e-2,
|
||||
)
|
||||
|
||||
|
||||
def _run_deferred_finalize_parity_test(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size: int,
|
||||
) -> None:
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("K3 latent-MoE tail fusion requires SM100")
|
||||
multi_process_parallel(
|
||||
monkeypatch,
|
||||
tp_size,
|
||||
1,
|
||||
_test_deferred_finalize_parity_worker,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_latent_moe_tail_tp8_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 8)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=16)
|
||||
def test_latent_moe_tail_tp16_matches_native_path(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_latent_moe_tail_test(monkeypatch, 16)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=8)
|
||||
def test_latent_moe_tail_deferred_finalize_matches_finalized(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
_run_deferred_finalize_parity_test(monkeypatch, 8)
|
||||
@@ -1,433 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""K3's fused chunked-context prefill must match the generic MLA impl.
|
||||
|
||||
The layer owns its own context loop so it can fuse the per-chunk K/V pack and
|
||||
skip re-quantizing an already-quantized query. That is only safe if it feeds the
|
||||
prefill backend exactly what ``MLACommonBaseImpl._compute_prefill_context``
|
||||
would, chunk for chunk.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLACommonBaseImpl,
|
||||
MLACommonImpl,
|
||||
MLACommonPrefillMetadata,
|
||||
build_mla_chunked_context_metadata,
|
||||
)
|
||||
from vllm.models.kimi_k3.nvidia.mla import MultiHeadLatentAttention
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Kimi-K3 fused MLA requires CUDA"
|
||||
)
|
||||
|
||||
_KV_LORA_RANK = 512
|
||||
_QK_NOPE = 128
|
||||
_QK_ROPE = 64
|
||||
_V_HEAD_DIM = 128
|
||||
_ENTRY = _KV_LORA_RANK + _QK_ROPE
|
||||
_NUM_HEADS = 2
|
||||
_BLOCK_SIZE = 16
|
||||
_WORKSPACE_TOKENS = 128
|
||||
# Splits one long request across chunks, packs short ones together, and leaves
|
||||
# the last request without any context.
|
||||
_CONTEXT_LENS = [200, 48, 32, 0]
|
||||
_QUERY_LENS = [8, 4, 6, 5]
|
||||
|
||||
|
||||
class _RecordingPrefillBackend:
|
||||
"""Records what each chunk is asked to attend over.
|
||||
|
||||
``honors_out`` mimics the two backend families: one that writes into a
|
||||
caller-provided ``out`` (trtllm_ragged, flashinfer, tokenspeed) and one that
|
||||
always returns its own buffer (flash_attn with a padded V, aiter).
|
||||
"""
|
||||
|
||||
def __init__(self, honors_out: bool = False) -> None:
|
||||
self.calls: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
|
||||
self.out_destinations: list[torch.Tensor | None] = []
|
||||
self._honors_out = honors_out
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "recording"
|
||||
|
||||
def supports_out(self) -> bool:
|
||||
return self._honors_out
|
||||
|
||||
def run_prefill_context_chunk(self, *, chunk, q, k, v, out=None):
|
||||
self.calls.append((q.float().clone(), k.float().clone(), v.float().clone()))
|
||||
self.out_destinations.append(out)
|
||||
assert out is None or self._honors_out
|
||||
# Fold K/V into the partial so any packing difference shows up in the
|
||||
# merged context output, not just in the recorded calls.
|
||||
digest = (k.float().mean() + v.float().mean()).item()
|
||||
num_q = q.shape[0]
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
(num_q, _NUM_HEADS, _V_HEAD_DIM), device=q.device, dtype=torch.bfloat16
|
||||
)
|
||||
else:
|
||||
assert out.shape == (num_q, _NUM_HEADS, _V_HEAD_DIM)
|
||||
out.fill_(digest)
|
||||
lse = torch.full(
|
||||
(_NUM_HEADS, num_q),
|
||||
1.0 + chunk.index,
|
||||
device=q.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return out, lse
|
||||
|
||||
|
||||
class _KVBProj(torch.nn.Module):
|
||||
"""Stand-in for the layer's ``kv_b_proj`` (returns an (out, bias) tuple).
|
||||
|
||||
Enforces the same input contract as the real linear methods, because that is
|
||||
what decides whether the gathered latent needs a cast:
|
||||
|
||||
* an fp8 weight consumes the fp8 latent directly and dequantizes internally,
|
||||
so it takes fp8 or bf16;
|
||||
* a bf16 weight -- what a stock K3 checkpoint carries -- is a plain
|
||||
``F.linear`` and rejects anything but bf16, exactly as torch does.
|
||||
|
||||
Either weight dtype produces a bf16 output.
|
||||
"""
|
||||
|
||||
def __init__(self, device: torch.device, weight_dtype: torch.dtype) -> None:
|
||||
super().__init__()
|
||||
weight = (
|
||||
torch.randn(
|
||||
_NUM_HEADS * (_QK_NOPE + _V_HEAD_DIM),
|
||||
_KV_LORA_RANK,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
* 0.05
|
||||
)
|
||||
self.register_buffer("weight", weight.to(weight_dtype))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||
if self.weight.dtype == torch.bfloat16 and x.dtype != torch.bfloat16:
|
||||
raise RuntimeError(
|
||||
"a bfloat16 kv_b_proj cannot consume the gathered latent as "
|
||||
f"{x.dtype}; it must be cast first"
|
||||
)
|
||||
return torch.nn.functional.linear(
|
||||
x.to(torch.bfloat16), self.weight.to(torch.bfloat16)
|
||||
), None
|
||||
|
||||
|
||||
class _FusedLayer:
|
||||
"""Only the attributes K3's context loop reads."""
|
||||
|
||||
_compute_prefill_context = MultiHeadLatentAttention._compute_prefill_context
|
||||
_gather_context_latent = MultiHeadLatentAttention._gather_context_latent
|
||||
_attn_read_kv_cache = MultiHeadLatentAttention._attn_read_kv_cache
|
||||
|
||||
def __init__(self, kv_b_proj, kv_cache, kv_cache_dtype, k_scale) -> None:
|
||||
self.kv_b_proj = kv_b_proj
|
||||
self.kv_cache = kv_cache
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self._k_scale = k_scale
|
||||
self.kv_lora_rank = _KV_LORA_RANK
|
||||
self.num_local_heads = _NUM_HEADS
|
||||
self.qk_nope_head_dim = _QK_NOPE
|
||||
self.v_head_dim = _V_HEAD_DIM
|
||||
|
||||
|
||||
class _ReferenceImpl:
|
||||
"""Only the attributes the generic context loop reads."""
|
||||
|
||||
_compute_prefill_context = MLACommonBaseImpl._compute_prefill_context
|
||||
_concat_k_nope_k_pe = MLACommonBaseImpl._concat_k_nope_k_pe
|
||||
_use_flashinfer_concat_mla_k = False
|
||||
|
||||
def __init__(self, kv_b_proj, kv_cache_dtype) -> None:
|
||||
self.kv_b_proj = kv_b_proj
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.kv_lora_rank = _KV_LORA_RANK
|
||||
self.num_heads = _NUM_HEADS
|
||||
self.qk_nope_head_dim = _QK_NOPE
|
||||
self.qk_rope_head_dim = _QK_ROPE
|
||||
self.v_head_dim = _V_HEAD_DIM
|
||||
|
||||
|
||||
def _build_prefill_metadata(
|
||||
device: torch.device,
|
||||
workspace_dtype: torch.dtype,
|
||||
q_data_type: torch.dtype,
|
||||
backend: _RecordingPrefillBackend,
|
||||
) -> MLACommonPrefillMetadata:
|
||||
query_start_loc_cpu = torch.zeros(len(_QUERY_LENS) + 1, dtype=torch.int32)
|
||||
query_start_loc_cpu[1:] = torch.tensor(_QUERY_LENS, dtype=torch.int32).cumsum(0)
|
||||
workspace = torch.empty(
|
||||
(_WORKSPACE_TOKENS, _ENTRY), dtype=workspace_dtype, device=device
|
||||
)
|
||||
chunked_context = build_mla_chunked_context_metadata(
|
||||
context_lens_cpu=torch.tensor(_CONTEXT_LENS, dtype=torch.int32),
|
||||
prefill_query_start_loc_cpu=query_start_loc_cpu,
|
||||
chunked_prefill_workspace=workspace,
|
||||
chunked_prefill_workspace_size=_WORKSPACE_TOKENS,
|
||||
block_size=_BLOCK_SIZE,
|
||||
align_chunk_to_block=True,
|
||||
device=device,
|
||||
dcp_world_size=1,
|
||||
dcp_local_block_size=1,
|
||||
dcp_virtual_block_size=1,
|
||||
)
|
||||
assert chunked_context is not None
|
||||
assert len(chunked_context.chunks) > 1, "the batch must exercise accumulation"
|
||||
|
||||
max_blocks = (max(_CONTEXT_LENS) + max(_QUERY_LENS)) // _BLOCK_SIZE + 1
|
||||
num_blocks = max_blocks * len(_CONTEXT_LENS)
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(
|
||||
len(_CONTEXT_LENS), max_blocks
|
||||
)
|
||||
return MLACommonPrefillMetadata(
|
||||
block_table=block_table,
|
||||
query_start_loc=query_start_loc_cpu.to(device),
|
||||
max_query_len=max(_QUERY_LENS),
|
||||
chunked_context=chunked_context,
|
||||
q_data_type=q_data_type,
|
||||
output_dtype=torch.bfloat16,
|
||||
prefill_backend=backend,
|
||||
), num_blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("honors_out", [False, True], ids=["copy_out", "writes_out"])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_b_proj_quantized", [True, False], ids=["fp8_kv_b_proj", "bf16_kv_b_proj"]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_context_matches_generic_impl(
|
||||
kv_b_proj_quantized: bool, kv_cache_dtype: str, honors_out: bool
|
||||
) -> None:
|
||||
"""Cache dtype and ``kv_b_proj`` dtype vary independently.
|
||||
|
||||
A stock K3 checkpoint pairs a bf16 ``kv_b_proj`` with an fp8 cache, so the
|
||||
fused loop cannot assume the gathered latent is already in the dtype
|
||||
``kv_b_proj`` accepts.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
quantized = kv_cache_dtype == "fp8"
|
||||
# An fp8 cache is read by an fp8 query, so the workspace keeps the fp8
|
||||
# layout; a bf16 cache dequantizes into a bf16 workspace.
|
||||
q_data_type = fp8 if quantized else torch.bfloat16
|
||||
workspace_dtype = q_data_type
|
||||
|
||||
kv_b_proj = _KVBProj(
|
||||
device, weight_dtype=fp8 if kv_b_proj_quantized else torch.bfloat16
|
||||
)
|
||||
k_scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
backend_fused = _RecordingPrefillBackend(honors_out=honors_out)
|
||||
# The reference impl never passes `out`, so it always allocates its own.
|
||||
backend_ref = _RecordingPrefillBackend()
|
||||
prefill_fused, num_blocks = _build_prefill_metadata(
|
||||
device, workspace_dtype, q_data_type, backend_fused
|
||||
)
|
||||
prefill_ref, _ = _build_prefill_metadata(
|
||||
device, workspace_dtype, q_data_type, backend_ref
|
||||
)
|
||||
|
||||
cache = torch.randn(
|
||||
(num_blocks, _BLOCK_SIZE, _ENTRY), device=device, dtype=torch.bfloat16
|
||||
)
|
||||
kv_cache = cache.to(fp8) if quantized else cache
|
||||
q = (
|
||||
torch.randn(
|
||||
(sum(_QUERY_LENS), _NUM_HEADS, _QK_NOPE + _QK_ROPE),
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
* 0.2
|
||||
).to(q_data_type)
|
||||
|
||||
layer = _FusedLayer(kv_b_proj, kv_cache, kv_cache_dtype, k_scale)
|
||||
fused_out, fused_lse = layer._compute_prefill_context(
|
||||
q, SimpleNamespace(prefill=prefill_fused)
|
||||
)
|
||||
|
||||
impl = _ReferenceImpl(kv_b_proj, kv_cache_dtype)
|
||||
ref_out, ref_lse = impl._compute_prefill_context(
|
||||
q, kv_cache, SimpleNamespace(prefill=prefill_ref), k_scale
|
||||
)
|
||||
|
||||
assert len(backend_fused.calls) == len(backend_ref.calls)
|
||||
for chunk_idx, (fused_call, ref_call) in enumerate(
|
||||
zip(backend_fused.calls, backend_ref.calls, strict=True)
|
||||
):
|
||||
for name, fused_t, ref_t in zip(
|
||||
("q", "k", "v"), fused_call, ref_call, strict=True
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
fused_t,
|
||||
ref_t,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
msg=lambda m, n=name, i=chunk_idx: f"chunk {i} {n} differs: {m}",
|
||||
)
|
||||
torch.testing.assert_close(fused_out, ref_out, atol=0, rtol=0)
|
||||
torch.testing.assert_close(fused_lse, ref_lse, atol=0, rtol=0)
|
||||
|
||||
# Every chunk but the continuation should have been written in place, i.e.
|
||||
# straight into the returned accumulator, with no intermediate copy.
|
||||
wrote_in_place = [
|
||||
out is not None and out.data_ptr() == fused_out[chunk.token_slice].data_ptr()
|
||||
for out, chunk in zip(
|
||||
backend_fused.out_destinations,
|
||||
prefill_fused.chunked_context.chunks,
|
||||
strict=True,
|
||||
)
|
||||
]
|
||||
continuations = [c.is_continuation for c in prefill_fused.chunked_context.chunks]
|
||||
assert any(continuations), "the batch must exercise a continuation chunk"
|
||||
if honors_out:
|
||||
assert wrote_in_place == [not c for c in continuations]
|
||||
else:
|
||||
assert not any(wrote_in_place)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_context_rejects_an_unquantized_query() -> None:
|
||||
"""The fp8 query is produced by the new-token epilogue, not re-cast here."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
prefill, num_blocks = _build_prefill_metadata(
|
||||
device, fp8, fp8, _RecordingPrefillBackend()
|
||||
)
|
||||
layer = _FusedLayer(
|
||||
_KVBProj(device, weight_dtype=fp8),
|
||||
torch.zeros((num_blocks, _BLOCK_SIZE, _ENTRY), device=device, dtype=fp8),
|
||||
"fp8",
|
||||
torch.ones(1, dtype=torch.float32, device=device),
|
||||
)
|
||||
q = torch.zeros(
|
||||
(sum(_QUERY_LENS), _NUM_HEADS, _QK_NOPE + _QK_ROPE),
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
with pytest.raises(AssertionError, match="new-token epilogue"):
|
||||
layer._compute_prefill_context(q, SimpleNamespace(prefill=prefill))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Decode context parallelism: the generic impl keeps its loop (allgather +
|
||||
# reorg) and only the per-chunk K/V pack is handed to the layer's fused kernel.
|
||||
# --------------------------------------------------------------------------
|
||||
_DCP = 2
|
||||
_DCP_TOKENS = 48
|
||||
|
||||
|
||||
def _run_dcp_context_loop(monkeypatch, *, q_data_type, fused_mla_kv_concat_fn):
|
||||
"""Drive ``MLACommonImpl._context_parallel_compute_prefill_context`` on one
|
||||
chunk with the collective pieces (gather, allgather, reorg) stubbed out."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda")
|
||||
kv_b_proj = _KVBProj(device, weight_dtype=torch.bfloat16)
|
||||
workspace = torch.randn(
|
||||
(_DCP_TOKENS * (_DCP + 1), _ENTRY), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
impl = SimpleNamespace(
|
||||
kv_cache_dtype="auto",
|
||||
kv_lora_rank=_KV_LORA_RANK,
|
||||
qk_rope_head_dim=_QK_ROPE,
|
||||
qk_nope_head_dim=_QK_NOPE,
|
||||
v_head_dim=_V_HEAD_DIM,
|
||||
num_heads=_NUM_HEADS,
|
||||
kv_b_proj=kv_b_proj,
|
||||
_use_flashinfer_concat_mla_k=False,
|
||||
)
|
||||
impl._concat_k_nope_k_pe = MLACommonImpl._concat_k_nope_k_pe.__get__(impl)
|
||||
|
||||
monkeypatch.setattr(mla_attention.ops, "cp_gather_cache", lambda **_: None)
|
||||
monkeypatch.setattr(mla_attention, "_get_kv_b_proj_input_dtype", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
mla_attention,
|
||||
"reorg_kvcache",
|
||||
lambda kv_c, k_pe, **_: (kv_c[:_DCP_TOKENS].squeeze(1), k_pe[:_DCP_TOKENS]),
|
||||
)
|
||||
chunk = SimpleNamespace(
|
||||
index=0,
|
||||
padded_local_seq_lens=[_DCP_TOKENS],
|
||||
local_context_lens_allranks=[[_DCP_TOKENS // _DCP] * _DCP],
|
||||
padded_local_cu_seq_lens=torch.tensor([0, _DCP_TOKENS], device=device),
|
||||
padded_local_token_to_seq=torch.zeros(
|
||||
_DCP_TOKENS, dtype=torch.int32, device=device
|
||||
),
|
||||
local_starts=[0],
|
||||
num_local_context_tokens=_DCP_TOKENS,
|
||||
request_slice=slice(0, 1),
|
||||
num_requests=1,
|
||||
starts=torch.tensor([0], device=device),
|
||||
num_context_tokens=_DCP_TOKENS,
|
||||
max_seq_len=_DCP_TOKENS,
|
||||
token_slice=slice(0, 4),
|
||||
)
|
||||
backend = _RecordingPrefillBackend()
|
||||
attn_metadata = SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
prefill_backend=backend,
|
||||
chunked_context=SimpleNamespace(
|
||||
workspace=workspace,
|
||||
chunks=[chunk],
|
||||
empty_token_slices=False,
|
||||
dcp_manager=SimpleNamespace(kv_gather=lambda dst, src: None),
|
||||
),
|
||||
q_data_type=q_data_type,
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
|
||||
)
|
||||
)
|
||||
q = torch.randn(
|
||||
(4, _NUM_HEADS, _QK_NOPE + _QK_ROPE), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
out, lse = MLACommonImpl._context_parallel_compute_prefill_context(
|
||||
impl,
|
||||
q,
|
||||
torch.empty(0, device=device),
|
||||
attn_metadata,
|
||||
k_scale=torch.ones(1, device=device),
|
||||
dcp_world_size=_DCP,
|
||||
fused_mla_kv_concat_fn=fused_mla_kv_concat_fn,
|
||||
)
|
||||
return backend, out, lse
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@torch.inference_mode()
|
||||
def test_dcp_context_fused_pack_matches_generic_tail(
|
||||
monkeypatch, kv_cache_dtype: str
|
||||
) -> None:
|
||||
"""Under DCP the layer's fused K/V pack must feed the prefill backend exactly
|
||||
what the generic impl's cast + split + concat tail feeds it."""
|
||||
q_data_type = (
|
||||
current_platform.fp8_dtype() if kv_cache_dtype == "fp8" else torch.bfloat16
|
||||
)
|
||||
layer = SimpleNamespace(qk_nope_head_dim=_QK_NOPE, v_head_dim=_V_HEAD_DIM)
|
||||
fused_fn = MultiHeadLatentAttention._fused_mla_kv_concat.__get__(layer)
|
||||
|
||||
backend_ref, ref_out, ref_lse = _run_dcp_context_loop(
|
||||
monkeypatch, q_data_type=q_data_type, fused_mla_kv_concat_fn=None
|
||||
)
|
||||
backend_fused, fused_out, fused_lse = _run_dcp_context_loop(
|
||||
monkeypatch, q_data_type=q_data_type, fused_mla_kv_concat_fn=fused_fn
|
||||
)
|
||||
assert len(backend_ref.calls) == len(backend_fused.calls) == 1
|
||||
for name, fused_t, ref_t in zip(
|
||||
("q", "k", "v"), backend_fused.calls[0], backend_ref.calls[0], strict=True
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
fused_t, ref_t, atol=0, rtol=0, msg=lambda m, n=name: f"{n} differs: {m}"
|
||||
)
|
||||
torch.testing.assert_close(fused_out, ref_out, atol=0, rtol=0)
|
||||
torch.testing.assert_close(fused_lse, ref_lse, atol=0, rtol=0)
|
||||
@@ -1,31 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.model import KimiDecoderLayer
|
||||
|
||||
|
||||
class _ReturningSharedKDA(nn.Module):
|
||||
def __init__(self, result: torch.Tensor) -> None:
|
||||
super().__init__()
|
||||
self.result = result
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, positions: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return self.result
|
||||
|
||||
|
||||
def test_low_rank_kda_caller_returns_projection_storage_directly() -> None:
|
||||
hidden_states = torch.randn(4, 8)
|
||||
projected = torch.randn_like(hidden_states)
|
||||
layer = object.__new__(KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.self_attn = _ReturningSharedKDA(projected)
|
||||
|
||||
output = layer._run_self_attn(torch.arange(4), hidden_states)
|
||||
|
||||
assert output is projected
|
||||
assert output.data_ptr() == projected.data_ptr()
|
||||
@@ -1,504 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from math import lcm
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
# Official Kimi-K3 config with 8 layers and no weights.
|
||||
MODEL = "riverclouds/Kimi-K3-8L-dummy"
|
||||
# The recipe's DSpark draft, reading target layers that exist in MODEL.
|
||||
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
|
||||
|
||||
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
|
||||
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
|
||||
NUM_TURNS = 3
|
||||
FIRST_PROMPT_BLOCKS = 3
|
||||
UNALIGNED_TAIL = 37
|
||||
TURN_TOKENS = 777
|
||||
MAX_TOKENS = 8
|
||||
NUM_LOGPROBS = 5
|
||||
STARTUP_TIMEOUT = 1200
|
||||
|
||||
BASE_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--language-model-only",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--no-enable-flashinfer-autotune",
|
||||
"--enable-prompt-tokens-details",
|
||||
# Default max_num_seqs OOMs one GPU during warmup.
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
|
||||
)
|
||||
|
||||
SPEC_CONFIG = {
|
||||
"method": "dspark",
|
||||
"model": DRAFT,
|
||||
"num_speculative_tokens": 7,
|
||||
"draft_sample_method": "probabilistic",
|
||||
"rejection_sample_method": "block",
|
||||
}
|
||||
|
||||
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
|
||||
MOONCAKE = {
|
||||
"kv_connector": "MooncakeStoreConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_load_failure_policy": "recompute",
|
||||
"kv_connector_extra_config": {
|
||||
"load_async": True,
|
||||
"lookup_async": True,
|
||||
"enable_cross_layers_blocks": False,
|
||||
},
|
||||
}
|
||||
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
|
||||
OFFLOAD = {
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"cpu_bytes_to_use_per_rank": 32 << 30,
|
||||
"lazy_offload": False,
|
||||
},
|
||||
}
|
||||
NIXL_OFFLOAD = {
|
||||
"kv_connector": "MultiConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mode:
|
||||
# A finer prefix_match_unit enables partial hits inside a Mamba block.
|
||||
prefix_match_unit: int | None
|
||||
spec: bool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
name = "block" if self.prefix_match_unit is None else "partial"
|
||||
return f"{name}-dspark" if self.spec else name
|
||||
|
||||
|
||||
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Instance:
|
||||
tp: int = 1
|
||||
dp: int = 1
|
||||
dcp: int = 1
|
||||
kv_config: dict | None = None
|
||||
prefix_caching: bool = True
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return self.tp * self.dp
|
||||
|
||||
def args(self, mode: Mode) -> list[str]:
|
||||
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
|
||||
if mode.spec:
|
||||
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
|
||||
if self.dp > 1:
|
||||
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
|
||||
if self.dcp > 1:
|
||||
args += ["--decode-context-parallel-size", str(self.dcp)]
|
||||
if self.kv_config is not None:
|
||||
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
|
||||
if not self.prefix_caching:
|
||||
return args + ["--no-enable-prefix-caching"]
|
||||
args.append("--enable-prefix-caching")
|
||||
if mode.prefix_match_unit is not None:
|
||||
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
|
||||
return args
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Deployment:
|
||||
decode: Instance
|
||||
prefill: Instance | None = None
|
||||
# Reset GPU prefix caches each turn so hits must come from CPU.
|
||||
offload: bool = False
|
||||
|
||||
@property
|
||||
def instances(self) -> tuple[Instance, ...]:
|
||||
return (self.prefill, self.decode) if self.prefill else (self.decode,)
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return sum(i.num_gpus for i in self.instances)
|
||||
|
||||
|
||||
DEPLOYMENTS = {
|
||||
"plain": Deployment(Instance()),
|
||||
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
|
||||
"dcp2": Deployment(Instance(tp=2, dcp=2)),
|
||||
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
|
||||
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
|
||||
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
|
||||
"pd-tp2-dep2": Deployment(
|
||||
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
|
||||
prefill=Instance(tp=2, kv_config=NIXL),
|
||||
),
|
||||
"pd-offload": Deployment(
|
||||
Instance(kv_config=NIXL_OFFLOAD),
|
||||
prefill=Instance(kv_config=NIXL_OFFLOAD),
|
||||
offload=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _known_failure(name: str, mode: Mode) -> str | None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
if mode.spec and deployment.decode.dcp > 1:
|
||||
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
|
||||
return None
|
||||
|
||||
|
||||
def _case(name: str, mode: Mode):
|
||||
marks = []
|
||||
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
|
||||
marks += multi_gpu_marks(num_gpus=num_gpus)
|
||||
if reason := _known_failure(name, mode):
|
||||
marks.append(pytest.mark.xfail(reason=reason, strict=True))
|
||||
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
|
||||
|
||||
|
||||
def _block_size(url: str) -> int:
|
||||
response = requests.get(f"{url}/metrics", timeout=30)
|
||||
response.raise_for_status()
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:cache_config_info":
|
||||
return lcm(
|
||||
*(
|
||||
int(value)
|
||||
for key in ("block_size", "mamba_block_size")
|
||||
if (value := sample.labels.get(key, "None")) != "None"
|
||||
)
|
||||
)
|
||||
raise AssertionError("missing vllm:cache_config_info metric")
|
||||
|
||||
|
||||
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
|
||||
body = {
|
||||
"model": MODEL,
|
||||
"prompt": prompt,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"logprobs": NUM_LOGPROBS,
|
||||
"return_tokens_as_token_ids": True,
|
||||
"cache_salt": salt,
|
||||
**extra,
|
||||
}
|
||||
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _pd_complete(
|
||||
prefill_url: str, decode_url: str, prompt: list[int], salt: str
|
||||
) -> tuple[dict, dict]:
|
||||
remote_decode = {
|
||||
"do_remote_decode": True,
|
||||
"do_remote_prefill": False,
|
||||
"remote_engine_id": None,
|
||||
"remote_block_ids": None,
|
||||
"remote_host": None,
|
||||
"remote_port": None,
|
||||
}
|
||||
prefill = _complete(
|
||||
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
|
||||
)
|
||||
decode = _complete(
|
||||
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
|
||||
)
|
||||
return prefill, decode
|
||||
|
||||
|
||||
def _cached(response: dict) -> int:
|
||||
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
|
||||
|
||||
def _token_id(token: str) -> int:
|
||||
return int(token.removeprefix("token_id:"))
|
||||
|
||||
|
||||
def _tokens_text_logprobs(response: dict):
|
||||
choice = response["choices"][0]
|
||||
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
|
||||
top = [
|
||||
{_token_id(t): lp for t, lp in d.items()}
|
||||
for d in choice["logprobs"]["top_logprobs"]
|
||||
]
|
||||
return ids, choice["text"], top
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(url: str) -> None:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
if response.json()["success"]:
|
||||
return
|
||||
assert time.monotonic() < deadline, "GPU cache reset timed out"
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Turn:
|
||||
prompt_len: int
|
||||
cached: int
|
||||
expected_cached: int
|
||||
recompute_cached: int
|
||||
output: tuple
|
||||
recompute: tuple
|
||||
|
||||
@property
|
||||
def matches_recompute(self) -> bool:
|
||||
return self.output[0] == self.recompute[0]
|
||||
|
||||
|
||||
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
|
||||
"""A turn must reuse the previous prompt up to its last checkpoint.
|
||||
|
||||
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
|
||||
"""
|
||||
if not prev_prompt_len:
|
||||
return 0
|
||||
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
|
||||
return max(checkpoint - hit_unit, 0) if spec else checkpoint
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
|
||||
with contextlib.ExitStack() as stack:
|
||||
# Sequential shutdown waits on memory still held by sibling servers.
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
next_gpu = 0
|
||||
for instance in deployment.instances:
|
||||
env = {}
|
||||
if deployment.offload:
|
||||
# Enables /reset_prefix_cache.
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1"
|
||||
if deployment.prefill is not None:
|
||||
gpus = range(next_gpu, next_gpu + instance.num_gpus)
|
||||
next_gpu += instance.num_gpus
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
|
||||
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
|
||||
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
|
||||
servers.append(
|
||||
RemoteOpenAIServer(
|
||||
MODEL,
|
||||
instance.args(mode),
|
||||
env_dict=env,
|
||||
max_wait_seconds=STARTUP_TIMEOUT,
|
||||
)
|
||||
)
|
||||
yield servers
|
||||
|
||||
|
||||
def _run_conversation(
|
||||
deployment: Deployment,
|
||||
servers: list[RemoteOpenAIServer],
|
||||
mode: Mode,
|
||||
) -> list[Turn]:
|
||||
# Under P/D the prefiller computes the prompt, so hits are read from it.
|
||||
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
|
||||
block_size = _block_size(compute_url)
|
||||
hit_unit = mode.prefix_match_unit or block_size
|
||||
rng = random.Random(0)
|
||||
|
||||
def new_tokens(n: int) -> list[int]:
|
||||
return [rng.randint(1000, 150000) for _ in range(n)]
|
||||
|
||||
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
|
||||
prev_len = 0
|
||||
turns = []
|
||||
for turn in range(NUM_TURNS):
|
||||
if deployment.offload:
|
||||
for server in servers:
|
||||
_reset_gpu_prefix_cache(server.url_root)
|
||||
if deployment.prefill is not None:
|
||||
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
|
||||
else:
|
||||
computed = output = _complete(decode_url, prompt, "conv")
|
||||
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
|
||||
turns.append(
|
||||
Turn(
|
||||
prompt_len=len(prompt),
|
||||
cached=_cached(computed),
|
||||
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
|
||||
recompute_cached=_cached(recompute),
|
||||
output=_tokens_text_logprobs(output),
|
||||
recompute=_tokens_text_logprobs(recompute),
|
||||
)
|
||||
)
|
||||
prev_len = len(prompt)
|
||||
prompt = prompt + new_tokens(TURN_TOKENS)
|
||||
return turns
|
||||
|
||||
|
||||
def _format(turns: list[Turn]) -> str:
|
||||
rows = ["turn prompt cached expected matches_recompute"]
|
||||
rows += [
|
||||
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
|
||||
f"{t.matches_recompute}"
|
||||
for i, t in enumerate(turns)
|
||||
]
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def _check_resend_reuses_prompt_checkpoint(
|
||||
deployment: Deployment, prompt_len: int
|
||||
) -> None:
|
||||
"""Resends reuse the Mamba checkpoint and its companion EAGLE attention proof."""
|
||||
mode = Mode(128, True)
|
||||
salt = str(uuid4())
|
||||
rng = random.Random(0)
|
||||
prompt = [rng.randint(1000, 150000) for _ in range(prompt_len)]
|
||||
with _serve(deployment, mode) as servers:
|
||||
url = servers[0].url_root
|
||||
cold = _complete(url, prompt, salt, max_tokens=1)
|
||||
if deployment.offload:
|
||||
_reset_gpu_prefix_cache(url)
|
||||
resend = _complete(url, prompt, salt, max_tokens=1)
|
||||
recompute = _complete(url, prompt, f"{salt}-recompute", max_tokens=1)
|
||||
|
||||
# EAGLE's dropped hash unit already leaves tokens to recompute on resend.
|
||||
expected = (prompt_len // 128 - 1) * 128
|
||||
actual = _cached(resend)
|
||||
assert _cached(cold) == _cached(recompute) == 0
|
||||
assert actual == expected, f"{prompt_len=}: {actual=}, {expected=}"
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[_tokens_text_logprobs(recompute)],
|
||||
outputs_1_lst=[_tokens_text_logprobs(resend)],
|
||||
name_0="recompute",
|
||||
name_1="resend",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prompt_len",
|
||||
[
|
||||
# Blocks=6144, PMU=128. P=7040 is PMU-aligned but not block-aligned:
|
||||
# ensure lookup reads the cached attention proof at 7040 despite
|
||||
# the final hit limit of 7039, then applies EAGLE's 128-token drop.
|
||||
pytest.param(7040, id="eagle-proof-scan"),
|
||||
# Blocks=6144, PMU=128. P=7296 is an exact PMU multiple:
|
||||
# ensure the Mamba checkpoint is 7168, not 7040 from applying both
|
||||
# the P - 1 cap and EAGLE's 128-token drop.
|
||||
pytest.param(7296, id="eagle-double-cap"),
|
||||
],
|
||||
)
|
||||
def test_dspark_exact_resend_reuses_prompt_checkpoint(prompt_len: int) -> None:
|
||||
_check_resend_reuses_prompt_checkpoint(DEPLOYMENTS["plain"], prompt_len)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mooncake_store(tmp_path, monkeypatch) -> Iterator[None]:
|
||||
"""Run a Mooncake master and point the servers' store config at it."""
|
||||
if shutil.which("mooncake_master") is None:
|
||||
pytest.skip("mooncake_master is not installed")
|
||||
port = get_open_port()
|
||||
log_path = tmp_path / "mooncake_master.log"
|
||||
with open(log_path, "w") as log:
|
||||
master = subprocess.Popen(
|
||||
[
|
||||
"mooncake_master",
|
||||
f"--port={port}",
|
||||
f"--metrics_port={get_open_port()}",
|
||||
"--enable_metric_reporting=false",
|
||||
"--default_kv_lease_ttl=60000",
|
||||
],
|
||||
stdout=log,
|
||||
stderr=subprocess.STDOUT,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
assert master.poll() is None, log_path.read_text()
|
||||
with (
|
||||
contextlib.suppress(OSError),
|
||||
socket.create_connection(("127.0.0.1", port)),
|
||||
):
|
||||
break
|
||||
assert time.monotonic() < deadline, "Mooncake master did not start"
|
||||
time.sleep(0.1)
|
||||
config_path = tmp_path / "mooncake.json"
|
||||
config_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"metadata_server": "P2PHANDSHAKE",
|
||||
"master_server_address": f"127.0.0.1:{port}",
|
||||
"protocol": "rdma",
|
||||
"device_name": os.getenv("MOONCAKE_DEVICE_NAME", "mlx5_12"),
|
||||
"global_segment_size": "4GB",
|
||||
"local_buffer_size": "1GB",
|
||||
}
|
||||
)
|
||||
)
|
||||
monkeypatch.setenv("MOONCAKE_CONFIG_PATH", str(config_path))
|
||||
monkeypatch.setenv("MC_MAX_MR_SIZE", str(4 << 30))
|
||||
yield
|
||||
finally:
|
||||
master.kill()
|
||||
master.wait()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("mooncake_store")
|
||||
def test_mooncake_resend_reuses_prompt_checkpoint() -> None:
|
||||
"""Blocks=6144, PMU=128. P=7449 needs Mamba state at 7296 and attention proof
|
||||
at 7424, both beyond the normal Mooncake save boundary of 6144: ensure they
|
||||
remain reusable from the store after a GPU cache reset."""
|
||||
_check_resend_reuses_prompt_checkpoint(
|
||||
Deployment(Instance(kv_config=MOONCAKE), offload=True), 7449
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
|
||||
)
|
||||
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
with _serve(deployment, mode) as servers:
|
||||
turns = _run_conversation(deployment, servers, mode)
|
||||
table = _format(turns)
|
||||
print(f"\n{name}-{mode.name}\n{table}")
|
||||
|
||||
assert all(t.recompute_cached == 0 for t in turns), table
|
||||
assert all(t.cached >= t.expected_cached for t in turns), table
|
||||
# Dummy weights differ across TP sizes.
|
||||
if len({i.tp for i in deployment.instances}) == 1:
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[t.recompute for t in turns],
|
||||
outputs_1_lst=[t.output for t in turns],
|
||||
name_0="recompute",
|
||||
name_1=name,
|
||||
)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Align-mode RecoverSSM state columns at recurrent block boundaries.
|
||||
|
||||
A speculative window whose accepted tokens end exactly on a recurrent block
|
||||
boundary must keep its state in the block holding the last committed token.
|
||||
The following block can still be unallocated, so selecting it would drop the
|
||||
commit while the bookkeeping recorded the state as advanced.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.kimi_k3.nvidia.ops.recoverssm import _prepare_commit_plan_kernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.worker.gpu.model_states.recoverssm import (
|
||||
_postprocess_recoverssm_align_kernel,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="The RecoverSSM kernels require a CUDA-alike device.",
|
||||
)
|
||||
|
||||
BLOCK_SIZE = 2048
|
||||
SPEC_QUERY_LEN = 4
|
||||
BLOCKS = [100, 101, 102, 103, 104, 105]
|
||||
|
||||
# (computed before the step, committed tokens, allocated block columns,
|
||||
# final column, boundary column or None)
|
||||
CASES = [
|
||||
# 8,188 + 4 ends exactly on the 8,192 boundary; column 4 is unallocated.
|
||||
(8188, 4, 4, 3, 3),
|
||||
# The window crosses the boundary: state moves on, checkpoint stays behind.
|
||||
(8190, 4, 5, 4, 3),
|
||||
# The window stays inside column 3.
|
||||
(8180, 4, 4, 3, None),
|
||||
# The first token after a boundary belongs to the new column.
|
||||
(8192, 1, 5, 4, None),
|
||||
# Single-token commit ending on the first boundary.
|
||||
(2047, 1, 1, 0, 0),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||
)
|
||||
def test_commit_plan_keeps_boundary_state_in_last_computed_block(
|
||||
num_computed, commit_len, allocated, final_col, boundary_col
|
||||
):
|
||||
device = current_platform.device_type
|
||||
columns = [
|
||||
block if col < allocated else NULL_BLOCK_ID for col, block in enumerate(BLOCKS)
|
||||
]
|
||||
block_table = torch.tensor([columns], dtype=torch.int32, device=device)
|
||||
num_accepted = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||
state_indices = torch.tensor(
|
||||
[BLOCKS[(num_computed - 1) // BLOCK_SIZE]], dtype=torch.int32, device=device
|
||||
)
|
||||
query_start_loc = torch.tensor(
|
||||
[0, SPEC_QUERY_LEN], dtype=torch.int32, device=device
|
||||
)
|
||||
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||
commit_lens, final_indices, boundary_indices, recovery_lens = (
|
||||
torch.full((1,), -7, dtype=torch.int32, device=device) for _ in range(4)
|
||||
)
|
||||
|
||||
_prepare_commit_plan_kernel[(1,)](
|
||||
num_accepted,
|
||||
None,
|
||||
state_indices,
|
||||
query_start_loc,
|
||||
block_table,
|
||||
computed,
|
||||
commit_lens,
|
||||
final_indices,
|
||||
boundary_indices,
|
||||
recovery_lens,
|
||||
NULL_BLOCK_ID,
|
||||
BLOCK_SIZE,
|
||||
block_table.shape[1],
|
||||
num_accepted.stride(0),
|
||||
0,
|
||||
state_indices.stride(0),
|
||||
query_start_loc.stride(0),
|
||||
block_table.stride(0),
|
||||
block_table.stride(1),
|
||||
computed.stride(0),
|
||||
SPEC_QUERY_LEN=SPEC_QUERY_LEN,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
assert commit_lens.item() == commit_len
|
||||
assert final_indices.item() == BLOCKS[final_col]
|
||||
expected_boundary = NULL_BLOCK_ID if boundary_col is None else BLOCKS[boundary_col]
|
||||
assert boundary_indices.item() == expected_boundary
|
||||
if boundary_col is not None:
|
||||
next_boundary = (num_computed // BLOCK_SIZE + 1) * BLOCK_SIZE
|
||||
assert recovery_lens.item() == next_boundary - num_computed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_computed,commit_len,allocated,final_col,boundary_col", CASES
|
||||
)
|
||||
def test_postprocess_records_the_committed_state_column(
|
||||
num_computed, commit_len, allocated, final_col, boundary_col
|
||||
):
|
||||
device = current_platform.device_type
|
||||
idx_mapping = torch.tensor([2], dtype=torch.int32, device=device)
|
||||
num_sampled = torch.tensor([commit_len], dtype=torch.int32, device=device)
|
||||
computed = torch.tensor([num_computed], dtype=torch.int32, device=device)
|
||||
state_idx = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||
num_accepted = torch.full((4,), -7, dtype=torch.int32, device=device)
|
||||
|
||||
_postprocess_recoverssm_align_kernel[(1,)](
|
||||
idx_mapping,
|
||||
num_sampled,
|
||||
None,
|
||||
computed,
|
||||
state_idx,
|
||||
num_accepted,
|
||||
MAMBA_BLOCK_SIZE=BLOCK_SIZE,
|
||||
BLOCK_TABLE_WIDTH=len(BLOCKS),
|
||||
)
|
||||
|
||||
assert state_idx[2].item() == final_col
|
||||
assert num_accepted[2].item() == 1
|
||||
assert state_idx[[0, 1, 3]].tolist() == [-7, -7, -7]
|
||||
@@ -1,461 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from types import MethodType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.models.common.ops import sequence_parallel as sp_ops
|
||||
from vllm.models.kimi_k3.nvidia import model as kimi_model
|
||||
from vllm.models.kimi_k3.nvidia import mtp as kimi_mtp
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class _IdentityNorm(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size), requires_grad=False)
|
||||
self.variance_epsilon = 1e-5
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
):
|
||||
if residual is None:
|
||||
return hidden_states
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
class _RecordingMoE(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.num_tokens = 0
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
self.num_tokens = hidden_states.shape[0]
|
||||
return hidden_states
|
||||
|
||||
|
||||
class _Projection(nn.Module):
|
||||
def __init__(self, hidden_size: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.ones(1, hidden_size),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
|
||||
class _SequenceParallelMTPBlock:
|
||||
use_sequence_parallel = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
):
|
||||
assert residual is None
|
||||
return hidden_states * 2, None, hidden_states * 3
|
||||
|
||||
|
||||
def _mock_sequence_parallel_collectives(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_reduce_scatter",
|
||||
lambda tensor: tensor.chunk(2, dim=0)[0],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "is_padding", "tp_rank", "expected"),
|
||||
[
|
||||
(1, None, 0, [False]),
|
||||
(1, None, 1, [True]),
|
||||
(5, None, 2, [False, True]),
|
||||
(5, None, 3, [True, True]),
|
||||
(5, [False, True, False, False, False], 0, [False, True]),
|
||||
],
|
||||
)
|
||||
def test_sp_padding_mask_marks_added_rows(
|
||||
monkeypatch,
|
||||
num_tokens: int,
|
||||
is_padding: list[bool] | None,
|
||||
tp_rank: int,
|
||||
expected: list[bool],
|
||||
):
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_world_size", lambda: 4)
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
|
||||
hidden_states = torch.empty(num_tokens, 2)
|
||||
padding = torch.tensor(is_padding) if is_padding is not None else None
|
||||
actual = sp_ops.sp_padding_mask(padding, hidden_states)
|
||||
|
||||
torch.testing.assert_close(actual, torch.tensor(expected))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data_parallel_size", "expected"),
|
||||
[
|
||||
(1, False),
|
||||
(2, True),
|
||||
],
|
||||
)
|
||||
def test_moe_sequence_parallel_requires_data_parallel(
|
||||
monkeypatch,
|
||||
data_parallel_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=data_parallel_size,
|
||||
enable_expert_parallel=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
)
|
||||
|
||||
assert parallel_config.use_sequence_parallel_moe is expected
|
||||
|
||||
|
||||
def test_kimi_decoder_layer_keeps_moe_states_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = False
|
||||
layer.use_sequence_parallel = True
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
|
||||
positions = torch.arange(3)
|
||||
full_hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
hidden_states = kimi_model.sp_shard(full_hidden_states)
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
hidden_states, prefix_sum, residual = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
assert prefix_sum is None
|
||||
assert hidden_states.shape == residual.shape == (2, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_attn_residual_states_stay_sequence_sharded(monkeypatch):
|
||||
layer = object.__new__(kimi_model.KimiDecoderLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.use_attn_res = True
|
||||
layer.use_sequence_parallel = True
|
||||
layer.prev_valid_blocks = 0
|
||||
layer.block_write_idx = 0
|
||||
layer.is_block_write_layer = False
|
||||
layer.input_layernorm = _IdentityNorm()
|
||||
layer.post_attention_layernorm = _IdentityNorm()
|
||||
layer.self_attention_res_norm = _IdentityNorm()
|
||||
layer.mlp_res_norm = _IdentityNorm()
|
||||
layer.self_attention_res_proj = _Projection()
|
||||
layer.mlp_res_proj = _Projection()
|
||||
layer.mlp = _RecordingMoE()
|
||||
layer._run_self_attn = MethodType(
|
||||
lambda self, positions, hidden_states: hidden_states,
|
||||
layer,
|
||||
)
|
||||
|
||||
_mock_sequence_parallel_collectives(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
kimi_model,
|
||||
"attn_res",
|
||||
lambda prefix_sum, hidden_states, *args, **kwargs: (
|
||||
prefix_sum if hidden_states is None else prefix_sum + hidden_states
|
||||
),
|
||||
)
|
||||
|
||||
prefix_sum = kimi_model.sp_shard(torch.arange(6, dtype=torch.float32).view(3, 2))
|
||||
block_residual = torch.zeros(2, 1, 2)
|
||||
hidden_states, prefix_sum, block_residual = layer(
|
||||
positions=torch.arange(3),
|
||||
hidden_states=None,
|
||||
prefix_sum=prefix_sum,
|
||||
residual=block_residual,
|
||||
)
|
||||
|
||||
assert hidden_states.shape == prefix_sum.shape == (2, 2)
|
||||
assert block_residual.shape == (2, 1, 2)
|
||||
assert layer.mlp.num_tokens == 2
|
||||
|
||||
|
||||
def test_kimi_mtp_restores_sequence_parallel_output(monkeypatch):
|
||||
layer = object.__new__(kimi_mtp.KimiK3MultiTokenPredictorLayer)
|
||||
nn.Module.__init__(layer)
|
||||
layer.enorm = _IdentityNorm()
|
||||
layer.hnorm = _IdentityNorm()
|
||||
layer.eh_proj = nn.Identity()
|
||||
object.__setattr__(layer, "mtp_block", _SequenceParallelMTPBlock())
|
||||
|
||||
final_norm = Mock(side_effect=lambda hidden_states: hidden_states + 1)
|
||||
object.__setattr__(
|
||||
layer,
|
||||
"shared_head",
|
||||
SimpleNamespace(norm=final_norm),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"fused_mtp_input",
|
||||
lambda positions, inputs_embeds, *args: inputs_embeds,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_shard",
|
||||
lambda tensor: torch.nn.functional.pad(tensor, (0, 0, 0, 1))[:2],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
kimi_mtp,
|
||||
"sp_all_gather",
|
||||
lambda tensor: torch.cat([tensor, tensor], dim=0),
|
||||
)
|
||||
|
||||
inputs_embeds = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
logits_hidden_states, hidden_states = layer(
|
||||
input_ids=torch.zeros(3, dtype=torch.long),
|
||||
positions=torch.arange(3),
|
||||
previous_hidden_states=torch.zeros_like(inputs_embeds),
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
|
||||
sharded_states = torch.nn.functional.pad(inputs_embeds, (0, 0, 0, 1))[:2]
|
||||
expected_hidden_states = torch.cat(
|
||||
[sharded_states * 5, sharded_states * 5],
|
||||
dim=0,
|
||||
)[:3]
|
||||
torch.testing.assert_close(hidden_states, expected_hidden_states)
|
||||
torch.testing.assert_close(logits_hidden_states, expected_hidden_states + 1)
|
||||
final_norm.assert_called_once()
|
||||
torch.testing.assert_close(final_norm.call_args.args[0], expected_hidden_states)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "use_sequence_parallel", "eligible", "tp_size", "expected"),
|
||||
[
|
||||
(True, True, True, 8, True),
|
||||
(False, True, True, 8, False), # opt-in only
|
||||
(True, False, True, 8, False), # replication only exists under SP
|
||||
(True, True, False, 8, False), # FusedMoE path owns the reduction
|
||||
(True, True, True, 1, False), # nothing to shard
|
||||
(True, True, True, 5, False), # 6144 % 5 -- would fail divide()
|
||||
],
|
||||
)
|
||||
def test_shard_sequence_parallel_mlp_gating(
|
||||
monkeypatch,
|
||||
enabled: bool,
|
||||
use_sequence_parallel: bool,
|
||||
eligible: bool,
|
||||
tp_size: int,
|
||||
expected: bool,
|
||||
):
|
||||
monkeypatch.setattr(kimi_model.envs, "VLLM_KIMI_K3_SHARD_SP_SHARED_EXPERT", enabled)
|
||||
monkeypatch.setattr(
|
||||
kimi_model, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
|
||||
assert (
|
||||
kimi_model.shard_sequence_parallel_mlp(
|
||||
hidden_size=7168,
|
||||
intermediate_size=6144,
|
||||
use_sequence_parallel=use_sequence_parallel,
|
||||
eligible=eligible,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("moe_backend", "all2all_backend", "expected"),
|
||||
[
|
||||
("deep_gemm_mega_moe", "naive", True),
|
||||
("auto", "deepep_v2", True),
|
||||
("auto", "deepep_low_latency", False),
|
||||
],
|
||||
)
|
||||
def test_shared_expert_sharding_backend_gate(
|
||||
moe_backend: str,
|
||||
all2all_backend: str,
|
||||
expected: bool,
|
||||
):
|
||||
vllm_config = SimpleNamespace(
|
||||
kernel_config=SimpleNamespace(moe_backend=moe_backend),
|
||||
parallel_config=SimpleNamespace(all2all_backend=all2all_backend),
|
||||
)
|
||||
|
||||
assert kimi_model.can_shard_sequence_parallel_shared_expert(vllm_config) is expected
|
||||
|
||||
|
||||
def test_sharded_sequence_parallel_mlp_matches_replicated(default_vllm_config):
|
||||
"""Sharded SP MLP must reproduce the replicated result for every token.
|
||||
|
||||
Each rank owns a *disjoint* token shard, so a weight shard alone cannot
|
||||
finish a rank's own tokens: the ranks must gather the full token set,
|
||||
compute partial sums over their intermediate shard, and reduce-scatter.
|
||||
Splicing per-rank feature slices together instead silently mixes different
|
||||
tokens and produces plausible-looking garbage.
|
||||
"""
|
||||
tp_size, hidden, intermediate, tokens_per_rank = 4, 16, 12, 3
|
||||
torch.manual_seed(0)
|
||||
num_tokens = tp_size * tokens_per_rank
|
||||
x = torch.randn(num_tokens, hidden)
|
||||
gate_weight = torch.randn(intermediate, hidden)
|
||||
up_weight = torch.randn(intermediate, hidden)
|
||||
down_weight = torch.randn(hidden, intermediate)
|
||||
act_fn = SiluAndMul()
|
||||
|
||||
replicated = act_fn(x @ torch.cat([gate_weight, up_weight]).T) @ down_weight.T
|
||||
|
||||
shard = intermediate // tp_size
|
||||
# Every rank all-gathers the full token set, then computes its partial.
|
||||
partials = [
|
||||
act_fn(
|
||||
x
|
||||
@ torch.cat(
|
||||
[
|
||||
gate_weight[r * shard : (r + 1) * shard],
|
||||
up_weight[r * shard : (r + 1) * shard],
|
||||
]
|
||||
).T
|
||||
)
|
||||
@ down_weight[:, r * shard : (r + 1) * shard].T
|
||||
for r in range(tp_size)
|
||||
]
|
||||
reduced = torch.stack(partials).sum(0)
|
||||
# Reduce-scatter: rank r keeps only its own token shard.
|
||||
for r in range(tp_size):
|
||||
mine = reduced[r * tokens_per_rank : (r + 1) * tokens_per_rank]
|
||||
expected = replicated[r * tokens_per_rank : (r + 1) * tokens_per_rank]
|
||||
torch.testing.assert_close(mine, expected, atol=1e-5, rtol=1e-5)
|
||||
|
||||
|
||||
def test_sp_all_gather_uses_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
expected = torch.cat([hidden_states, hidden_states])
|
||||
custom_all_gather = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_all_gather=custom_all_gather,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", fallback)
|
||||
|
||||
output = sp_ops.sp_all_gather(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
custom_all_gather.assert_called_once_with(hidden_states)
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
def test_sp_reduce_scatter_uses_custom_kernel_after_padding(monkeypatch):
|
||||
hidden_states = torch.arange(6, dtype=torch.float32).view(3, 2)
|
||||
expected = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
custom_reduce_scatter = Mock(return_value=expected)
|
||||
device_communicator = SimpleNamespace(
|
||||
custom_reduce_scatter=custom_reduce_scatter,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=device_communicator),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
fallback = Mock(side_effect=AssertionError("unexpected fallback"))
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_reduce_scatter", fallback)
|
||||
|
||||
output = sp_ops.sp_reduce_scatter(hidden_states)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
padded = custom_reduce_scatter.call_args.args[0]
|
||||
assert padded.shape == (4, 2)
|
||||
torch.testing.assert_close(padded[:3], hidden_states)
|
||||
torch.testing.assert_close(padded[3], torch.zeros(2))
|
||||
fallback.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(3,), (3, 2, 2)])
|
||||
def test_sp_shard_pads_only_the_token_axis(monkeypatch, shape):
|
||||
hidden_states = torch.arange(math.prod(shape), dtype=torch.float32).view(shape)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
monkeypatch.setattr(sp_ops, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
|
||||
output = sp_ops.sp_shard(hidden_states)
|
||||
|
||||
padding = hidden_states.new_zeros((1, *shape[1:]))
|
||||
expected = torch.cat([hidden_states, padding])[2:]
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
def test_sp_collectives_fall_back_without_custom_kernel(monkeypatch):
|
||||
hidden_states = torch.arange(4, dtype=torch.float32).view(2, 2)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(device_communicator=None),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"get_tensor_model_parallel_world_size",
|
||||
lambda: 2,
|
||||
)
|
||||
all_gather = Mock(return_value=hidden_states)
|
||||
reduce_scatter = Mock(return_value=hidden_states)
|
||||
monkeypatch.setattr(sp_ops, "tensor_model_parallel_all_gather", all_gather)
|
||||
monkeypatch.setattr(
|
||||
sp_ops,
|
||||
"tensor_model_parallel_reduce_scatter",
|
||||
reduce_scatter,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(sp_ops.sp_all_gather(hidden_states), hidden_states)
|
||||
torch.testing.assert_close(sp_ops.sp_reduce_scatter(hidden_states), hidden_states)
|
||||
all_gather.assert_called_once_with(hidden_states, 0)
|
||||
reduce_scatter.assert_called_once_with(hidden_states, 0)
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models import gemma
|
||||
from vllm.model_executor.models.gemma3n import (
|
||||
Gemma3nTextModel,
|
||||
_kv_sharing_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4 import (
|
||||
Gemma4ForCausalLM,
|
||||
_gemma4_layer_weights_mapper,
|
||||
)
|
||||
from vllm.model_executor.models.gemma4_dspark import (
|
||||
Gemma4DSparkForCausalLM,
|
||||
Gemma4DSparkModel,
|
||||
)
|
||||
|
||||
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize(
|
||||
"enabled,has_weights,with_markov",
|
||||
[
|
||||
pytest.param(True, True, False, id="hidden-only"),
|
||||
pytest.param(True, True, True, id="with-markov"),
|
||||
pytest.param(True, False, True, id="missing-weights"),
|
||||
pytest.param(False, True, True, id="disabled-head"),
|
||||
],
|
||||
)
|
||||
def test_gemma4_dspark_loads_confidence_head(
|
||||
monkeypatch, enabled, has_weights, with_markov
|
||||
) -> None:
|
||||
"""Use checkpoint confidence parameters, or disable an unavailable head."""
|
||||
config = SimpleNamespace(
|
||||
vocab_size=64,
|
||||
hidden_size=8,
|
||||
target_layer_ids=[0, 1],
|
||||
num_hidden_layers=0,
|
||||
rms_norm_eps=1e-6,
|
||||
markov_rank=4,
|
||||
enable_confidence_head=enabled,
|
||||
confidence_head_with_markov=with_markov,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.NONE),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
speculative_config=SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(hf_config=config),
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(Gemma4DSparkModel, "_build_fused_kv_buffers", lambda _: None)
|
||||
model = Gemma4DSparkForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
width = config.hidden_size + (config.markov_rank if with_markov else 0)
|
||||
weight = torch.arange(width, dtype=torch.bfloat16).reshape(1, -1) / 16
|
||||
bias = torch.tensor([-0.5], dtype=torch.bfloat16)
|
||||
loaded = model.load_weights(
|
||||
[("confidence_head.proj.weight", weight), ("confidence_head.proj.bias", bias)]
|
||||
if has_weights
|
||||
else []
|
||||
)
|
||||
if not (enabled and has_weights):
|
||||
assert model.model.confidence_head is None
|
||||
assert not loaded
|
||||
return
|
||||
|
||||
assert loaded == {
|
||||
"model.confidence_head.proj.weight",
|
||||
"model.confidence_head.proj.bias",
|
||||
}
|
||||
assert model.model.confidence_head.proj.weight.dtype == torch.float32
|
||||
hidden = torch.full((3, config.hidden_size), 0.25, dtype=torch.bfloat16)
|
||||
markov = torch.full((3, config.markov_rank), -0.5, dtype=torch.bfloat16)
|
||||
inputs = torch.cat([hidden, markov], dim=-1) if with_markov else hidden
|
||||
expected = (inputs.float() @ weight.float().T + bias.float()).sigmoid().squeeze(-1)
|
||||
torch.testing.assert_close(model.compute_confidence(hidden, markov), expected)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
"""A physical LM head must load after checkpoint-driven untying."""
|
||||
|
||||
class StubGemmaModel(torch.nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix):
|
||||
super().__init__()
|
||||
self.embed_tokens = VocabParallelEmbedding(4, 2)
|
||||
self.make_empty_intermediate_tensors = None
|
||||
|
||||
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
|
||||
config = SimpleNamespace(
|
||||
vocab_size=4,
|
||||
hidden_size=2,
|
||||
tie_word_embeddings=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
)
|
||||
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
embedding_weight = torch.full((4, 2), 1.0)
|
||||
lm_head_weight = torch.full((4, 2), 2.0)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("model.embed_tokens.weight", embedding_weight),
|
||||
("lm_head.weight", lm_head_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
|
||||
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_attention_mapper() -> None:
|
||||
"""Layers with a qkv_proj pack q/k/v; `attention_k_eq_v` full-attention
|
||||
layers also load K as the V shard, and leave any v_proj such a checkpoint
|
||||
ships unmapped so it fails the load rather than silently overwriting V;
|
||||
KV-shared layers keep q_proj and drop the K/V tensors original checkpoints
|
||||
still ship for them."""
|
||||
config = SimpleNamespace(
|
||||
num_hidden_layers=3,
|
||||
num_kv_shared_layers=1,
|
||||
attention_k_eq_v=True,
|
||||
layer_types=["sliding_attention", "full_attention", "sliding_attention"],
|
||||
)
|
||||
weights = [
|
||||
(f"model.layers.{i}.self_attn.{tensor}.weight", torch.full((2, 2), i + 1.0))
|
||||
for i in range(3)
|
||||
for tensor in ("q_proj", "k_proj", "k_norm")
|
||||
] + [
|
||||
("model.layers.0.mlp.up_proj.weight", torch.empty(0)),
|
||||
("model.layers.1.self_attn.v_proj.weight", torch.empty(0)),
|
||||
]
|
||||
|
||||
mapper = _gemma4_layer_weights_mapper(config)
|
||||
mapped = list(mapper.apply(weights))
|
||||
|
||||
assert [(name, getattr(w, "shard_id", None)) for name, w in mapped] == [
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.0.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.0.self_attn.k_norm.weight", None),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "q"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "k"),
|
||||
("model.layers.1.self_attn.qkv_proj.weight", "v"),
|
||||
("model.layers.1.self_attn.k_norm.weight", None),
|
||||
("model.layers.2.self_attn.q_proj.weight", None),
|
||||
("model.layers.0.mlp.gate_up_proj.weight", 1),
|
||||
("model.layers.1.self_attn.v_proj.weight", None),
|
||||
]
|
||||
k_weight, v_weight = weights[4][1], mapped[5][1]
|
||||
assert torch.equal(v_weight, k_weight) and v_weight is not k_weight
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma4_expert_names_strip_language_model_prefix() -> None:
|
||||
"""The text-only path reuses the conditional wrapper's checkpoint naming,
|
||||
so fused and per-expert tensors reach the experts under `model.*`."""
|
||||
prefix = "model.language_model.layers.0."
|
||||
weights = [
|
||||
(prefix + name, torch.empty(0))
|
||||
for name in (
|
||||
"experts.gate_up_proj",
|
||||
"experts.3.down_proj.weight_packed",
|
||||
"router.per_expert_scale",
|
||||
)
|
||||
]
|
||||
|
||||
mapped = [
|
||||
(name, getattr(w, "shard_id", None))
|
||||
for name, w in Gemma4ForCausalLM.hf_to_vllm_mapper.apply(weights)
|
||||
]
|
||||
|
||||
assert mapped == [
|
||||
("model.layers.0.experts.gate_up_proj", None),
|
||||
("model.layers.0.experts.3.down_proj.weight_packed", None),
|
||||
("model.layers.0.router.per_expert_scale", None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_gemma3n_kv_shared_layer_mapper() -> None:
|
||||
"""Only non-shared layers pack q/k/v into qkv_proj; KV-shared layers keep
|
||||
q_proj and drop the redundant K/V tensors original checkpoints ship."""
|
||||
config = SimpleNamespace(num_hidden_layers=4, num_kv_shared_layers=2)
|
||||
mapper = Gemma3nTextModel.hf_to_vllm_mapper | _kv_sharing_weights_mapper(config)
|
||||
weights = [
|
||||
(f"layers.{i}.self_attn.{tensor}.weight", torch.empty(0))
|
||||
for i in (1, 3)
|
||||
for tensor in ("q_proj", "k_proj", "v_proj", "k_norm", "o_proj")
|
||||
]
|
||||
|
||||
mapped = [
|
||||
(name, getattr(weight, "shard_id", None))
|
||||
for name, weight in mapper.apply(weights)
|
||||
]
|
||||
|
||||
assert mapped == [
|
||||
("layers.1.self_attn.qkv_proj.weight", "q"),
|
||||
("layers.1.self_attn.qkv_proj.weight", "k"),
|
||||
("layers.1.self_attn.qkv_proj.weight", "v"),
|
||||
("layers.1.self_attn.k_norm.weight", None),
|
||||
("layers.1.self_attn.o_proj.weight", None),
|
||||
("layers.3.self_attn.q_proj.weight", None),
|
||||
("layers.3.self_attn.o_proj.weight", None),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
load_format="dummy",
|
||||
) as llm:
|
||||
if model == "google/gemma-3-4b-it":
|
||||
normalizers = llm.llm.collective_rpc(
|
||||
lambda self: (
|
||||
self.model_runner.model.language_model.model.normalizer.cpu().item()
|
||||
) # noqa: E501
|
||||
)
|
||||
config = llm.llm.llm_engine.model_config.hf_config.text_config
|
||||
else:
|
||||
normalizers = llm.llm.collective_rpc(
|
||||
lambda self: self.model_runner.model.model.normalizer.cpu().item()
|
||||
)
|
||||
config = llm.llm.llm_engine.model_config.hf_config
|
||||
assert np.allclose(normalizers, config.hidden_size**0.5, rtol=2e-3)
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from transformers import GraniteConfig
|
||||
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
MODELS = [
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
"ibm-granite/granite-swash-2b",
|
||||
"ibm-granite/granite-swash-3b-a600m",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.cpu_model
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(model, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def test_granite_swa_features_are_off_without_swa_config():
|
||||
"""A plain Granite config must not pick up sliding windows or sinks."""
|
||||
config = GraniteConfig(num_hidden_layers=4)
|
||||
theta = config.rope_parameters["rope_theta"]
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(4)] == [
|
||||
(None, theta, False)
|
||||
] * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attention_sinks, expected_sink", [(None, True), (False, False)]
|
||||
)
|
||||
def test_granite_swa_features_resolve_per_layer(attention_sinks, expected_sink):
|
||||
"""`layer_types`/`layer_rope_theta` apply per layer; theta 0 means NoPE."""
|
||||
kwargs = {} if attention_sinks is None else {"attention_sinks": attention_sinks}
|
||||
config = GraniteConfig(
|
||||
num_hidden_layers=3,
|
||||
sliding_window=128,
|
||||
layer_types=["full_attention", "sliding_attention", "sliding_attention"],
|
||||
layer_rope_theta=[10000.0, 0.0, 1000000.0],
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
assert [granite_layer_attn_params(config, i) for i in range(3)] == [
|
||||
(None, 10000.0, expected_sink),
|
||||
(128, 0.0, expected_sink),
|
||||
(128, 1000000.0, expected_sink),
|
||||
]
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
MODELS = [
|
||||
"microsoft/Phi-3.5-MoE-instruct",
|
||||
]
|
||||
|
||||
|
||||
def test_phimoe_routing_function():
|
||||
from vllm.model_executor.models.phimoe import phimoe_routing_function
|
||||
|
||||
test_case = {
|
||||
0: {
|
||||
"hidden_states": torch.tensor(
|
||||
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
|
||||
).view(4, 2),
|
||||
"gating_output": torch.tensor(
|
||||
[0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk": 2,
|
||||
"renormalize": False,
|
||||
},
|
||||
1: {
|
||||
"hidden_states": torch.tensor(
|
||||
[1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
|
||||
).view(4, 2),
|
||||
"gating_output": torch.tensor(
|
||||
[0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk": 2,
|
||||
"renormalize": False,
|
||||
},
|
||||
}
|
||||
|
||||
ground_truth = {
|
||||
0: {
|
||||
"topk_weights": torch.tensor(
|
||||
[1.0, 1.0], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
|
||||
},
|
||||
1: {
|
||||
"topk_weights": torch.tensor(
|
||||
[0.5, 1.0], dtype=torch.float32, requires_grad=False
|
||||
),
|
||||
"topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
|
||||
},
|
||||
}
|
||||
|
||||
for test_id in test_case:
|
||||
topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
|
||||
assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"])
|
||||
assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
|
||||
|
||||
|
||||
# There is a known issue that triggers `AttributeError: 'DynamicCache'
|
||||
# object has no attribute 'seen_tokens'` when running:
|
||||
# `tests/models/language/generation/test_phimoe.py::test_models
|
||||
# [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]`
|
||||
# This issue is being investigated and tracked in:
|
||||
# https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58
|
||||
# It is platform-agnostic. Therefore, we skip this test on all platforms for now.
|
||||
@pytest.mark.skip(
|
||||
reason="Skipping due to known issue: "
|
||||
"'DynamicCache' object has no attribute 'seen_tokens'. See: "
|
||||
"https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 "
|
||||
"for details.",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
condition=current_platform.is_cpu(),
|
||||
reason="This test takes a lot time to run on CPU, "
|
||||
"and vllm CI's disk space is not enough for this model.",
|
||||
)
|
||||
@large_gpu_test(min_gb=80)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(model, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pytest configuration for vLLM language generation tests."""
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def pytest_sessionstart(session):
|
||||
"""Configure ROCm-specific settings before test session starts."""
|
||||
if current_platform.is_rocm():
|
||||
torch.set_float32_matmul_precision("high")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def release_gpu_memory_between_tests():
|
||||
"""Reclaim cached GPU memory between tests in this shard.
|
||||
|
||||
These tests start many vLLM/HF engines sequentially in one pytest
|
||||
process; without explicit reclamation between tests, VRAM fragmentation
|
||||
accumulates and later tests fail at engine startup ("Engine core
|
||||
initialization failed").
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm():
|
||||
return
|
||||
try:
|
||||
wait_for_memory_to_settle()
|
||||
except ValueError as e:
|
||||
# Longer-lived (class/module-scoped) engine fixtures may legitimately
|
||||
# still hold VRAM; nothing further to reclaim in that case.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
|
||||
from tests.conftest import HfRunner
|
||||
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
|
||||
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
|
||||
|
||||
|
||||
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
|
||||
embeddings = []
|
||||
for d in response.data:
|
||||
assert isinstance(d.embedding, list)
|
||||
embeddings.append(d.embedding)
|
||||
return embeddings
|
||||
|
||||
|
||||
def run_embedding_correctness_test(
|
||||
hf_model: "HfRunner",
|
||||
inputs: list[str],
|
||||
vllm_outputs: Sequence[list[float]],
|
||||
dimensions: int | None = None,
|
||||
):
|
||||
hf_outputs = hf_model.encode(inputs)
|
||||
if dimensions:
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info: EmbedModelInfo,
|
||||
example_prompts,
|
||||
vllm_extra_kwargs=None,
|
||||
hf_model_callback=None,
|
||||
):
|
||||
pytest.skip("Debug only, ci prefers to use mteb test.")
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = model_info.dtype
|
||||
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=model_info.hf_dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
if hf_model_callback is not None:
|
||||
hf_model_callback(hf_model)
|
||||
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
async def run_client_embeddings(
|
||||
client: openai.AsyncOpenAI,
|
||||
model_name: str,
|
||||
queries: list[str],
|
||||
instruction: str = "",
|
||||
) -> list[list[float]]:
|
||||
outputs = await client.embeddings.create(
|
||||
model=model_name,
|
||||
input=[instruction + q for q in queries],
|
||||
)
|
||||
return [data.embedding for data in outputs.data]
|
||||
@@ -1,151 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from tests.utils import VLLM_PATH
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_decoder_token_embed_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch, model: str
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
chunk_size = 10
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_model_len=128,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
enforce_eager=True,
|
||||
# `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_embed(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
auto_cls=AutoModel,
|
||||
) as hf_model:
|
||||
hf_outputs = []
|
||||
for token_prompt in token_prompts:
|
||||
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
|
||||
input_ids = inputs["input_ids"]
|
||||
output = hf_model.model(input_ids)
|
||||
hf_outputs.append(output.last_hidden_state.cpu().float()[0])
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_output,
|
||||
embeddings_1_lst=vllm_output,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
_RERANKER_HF_OVERRIDES = {
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
_RERANKER_TEMPLATE = VLLM_PATH / "examples/pooling/score/template/qwen3_reranker.jinja"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Qwen/Qwen3-Reranker-0.6B"])
|
||||
@torch.inference_mode
|
||||
def test_last_pool_score_chunked_prefill_matches_unchunked(vllm_runner, model: str):
|
||||
"""LAST-pooling score must not depend on whether the prompt is chunked.
|
||||
|
||||
Regression test for a wrong-score bug where, under ``torch.compile`` (i.e.
|
||||
not ``enforce_eager``), a query+document pair long enough to be split across
|
||||
prefill chunks produced a corrupted last-token hidden state and thus a
|
||||
wrong relevance score, while the same input scored correctly unchunked.
|
||||
The existing pooling+chunked-prefill coverage runs ``enforce_eager=True``
|
||||
and so never exercised the compiled path.
|
||||
"""
|
||||
chat_template = _RERANKER_TEMPLATE.read_text()
|
||||
query = "What organelle produces energy in the cell?"
|
||||
# A long, matching document so query + doc exceeds the chunk size below.
|
||||
document = (
|
||||
"The mitochondria is the powerhouse of the cell. It generates most of "
|
||||
"the cell chemical energy through oxidative phosphorylation. "
|
||||
) * 400
|
||||
|
||||
def score_with(max_num_batched_tokens: int) -> float:
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
hf_overrides=_RERANKER_HF_OVERRIDES,
|
||||
max_model_len=16384,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
) as vllm_model:
|
||||
return vllm_model.score(query, document, chat_template=chat_template)[0]
|
||||
|
||||
# Chunk size forces the prompt across several prefill chunks; the large
|
||||
# budget keeps it in a single chunk as the reference.
|
||||
chunked = score_with(2048)
|
||||
unchunked = score_with(16384)
|
||||
|
||||
assert chunked == pytest.approx(unchunked, abs=5e-2), (
|
||||
f"chunked score {chunked} diverged from unchunked {unchunked}"
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode
|
||||
def test_sequence_embed_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "Qwen/Qwen3-Embedding-0.6B"
|
||||
chunk_size = 10
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in (25, 27)]
|
||||
prompts = [TokensPrompt(prompt_token_ids=t) for t in token_prompts]
|
||||
|
||||
with hf_runner(model, auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for token_prompt in token_prompts:
|
||||
inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
|
||||
output = hf_model.model(inputs["input_ids"])
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state.float()[0, -1], dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="embed"),
|
||||
max_model_len=64,
|
||||
max_num_batched_tokens=chunk_size,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["jason9693/Qwen2.5-1.5B-apeach"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [s * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
|
||||
) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
|
||||
# First Run
|
||||
vllm_model.classify(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
example_prompts, pooling_task="classify"
|
||||
)
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
assert torch.allclose(
|
||||
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-Embedding-0.6B"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
):
|
||||
# example_prompts is too short for testing prefix_caching
|
||||
example_prompts = [str(s).strip() * 10 for s in example_prompts]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert cache_config.enable_prefix_caching
|
||||
|
||||
# First Run
|
||||
vllm_model.embed(example_prompts)
|
||||
|
||||
# assert prefix_caching works
|
||||
pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
|
||||
for output in pooling_outputs:
|
||||
assert output.num_cached_tokens > 0
|
||||
vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/e5-small",
|
||||
"Alibaba-NLP/gte-Qwen2-1.5B-instruct", # is_causal == False
|
||||
"papluca/xlm-roberta-base-language-detection",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_non_causal_models(
|
||||
hf_runner, vllm_runner, example_prompts, model: str, dtype: str
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
cache_config = vllm_config.cache_config
|
||||
assert not cache_config.enable_prefix_caching
|
||||
@@ -1,212 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from ....utils import RemoteOpenAIServer
|
||||
from .embed_utils import run_client_embeddings
|
||||
|
||||
MODEL_NAME = "BAAI/bge-m3"
|
||||
MAX_MODEL_LEN = 512
|
||||
|
||||
|
||||
# Example from https://huggingface.co/BAAI/bge-m3
|
||||
sentences_1 = ["What is BGE M3?", "Definition of BM25"]
|
||||
sentences_2 = [
|
||||
"BGE M3 is an embedding model supporting dense retrieval, "
|
||||
"lexical matching and multi-vector interaction.",
|
||||
"BM25 is a bag-of-words retrieval function that ranks a set "
|
||||
"of documents based on the query terms appearing in each document",
|
||||
]
|
||||
|
||||
similarity_reference = [[0.6259, 0.3474], [0.3309, 0.6734]]
|
||||
lexical_score_reference = [0.19554901123046875, 0.0]
|
||||
colbert_score_reference = [0.7797, 0.4620]
|
||||
SUPPORTED_TASKS = ["embed", "token_embed", "token_classify"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=SUPPORTED_TASKS)
|
||||
def pooling_task(request):
|
||||
yield request.param
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(pooling_task):
|
||||
args = [
|
||||
"--max-model-len",
|
||||
str(MAX_MODEL_LEN),
|
||||
"--hf-overrides",
|
||||
'{"architectures": ["BgeM3EmbeddingModel"]}',
|
||||
"--pooler-config.task",
|
||||
pooling_task,
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_embedding(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "embed":
|
||||
with pytest.raises(openai.InternalServerError):
|
||||
await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_1,
|
||||
)
|
||||
return
|
||||
|
||||
embeddings_list_1 = await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_1,
|
||||
)
|
||||
embeddings_list_2 = await run_client_embeddings(
|
||||
client,
|
||||
MODEL_NAME,
|
||||
sentences_2,
|
||||
)
|
||||
|
||||
embeddings_1 = torch.tensor(embeddings_list_1)
|
||||
embeddings_2 = torch.tensor(embeddings_list_2)
|
||||
similarity = embeddings_1 @ embeddings_2.T
|
||||
|
||||
# reference values from BAAI/bge-m3 documentation
|
||||
reference = torch.tensor(similarity_reference)
|
||||
|
||||
assert torch.allclose(similarity, reference, rtol=0.01)
|
||||
|
||||
|
||||
async def tokenize(client: openai.AsyncOpenAI, sentences: list[str]) -> list[list[int]]:
|
||||
futures = []
|
||||
for sentence in sentences:
|
||||
futures.append(
|
||||
client.post(
|
||||
"../tokenize",
|
||||
body={"model": MODEL_NAME, "prompt": sentence},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
)
|
||||
return [(await future).json()["tokens"] for future in futures]
|
||||
|
||||
|
||||
async def sparse_embeddings(
|
||||
client: openai.AsyncOpenAI, sentences: list[str]
|
||||
) -> list[dict[int, float]]:
|
||||
all_tokens = await tokenize(client, sentences)
|
||||
result = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences, "task": "token_classify"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
all_embeddings = [data["data"] for data in result.json()["data"]]
|
||||
|
||||
ret = []
|
||||
|
||||
for sent_tokens, sent_emb in zip(all_tokens, all_embeddings):
|
||||
token_embs = dict[int, float]()
|
||||
if sent_tokens[0] == 0:
|
||||
sent_tokens = sent_tokens[1:]
|
||||
for token, val in zip(sent_tokens, sent_emb):
|
||||
token_embs[token] = max(val, token_embs.get(token, 0.0))
|
||||
ret.append(token_embs)
|
||||
return ret
|
||||
|
||||
|
||||
# Based on https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L129
|
||||
def compute_lexical_matching_score(
|
||||
lw1: dict[int, float], lw2: dict[int, float]
|
||||
) -> float:
|
||||
scores = 0.0
|
||||
for token, weight in lw1.items():
|
||||
if token in lw2:
|
||||
scores += weight * lw2[token]
|
||||
return scores
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_sparse_embedding(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "token_classify":
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await sparse_embeddings(client, sentences_1)
|
||||
return
|
||||
|
||||
embeddings_1 = await sparse_embeddings(client, sentences_1)
|
||||
embeddings_2 = await sparse_embeddings(client, sentences_2)
|
||||
|
||||
lexical_scores_1_0_x_2_0 = compute_lexical_matching_score(
|
||||
embeddings_1[0], embeddings_2[0]
|
||||
)
|
||||
assert lexical_scores_1_0_x_2_0 == pytest.approx(
|
||||
lexical_score_reference[0], rel=0.01
|
||||
)
|
||||
|
||||
lexical_scores_1_0_x_1_1 = compute_lexical_matching_score(
|
||||
embeddings_1[0], embeddings_1[1]
|
||||
)
|
||||
assert lexical_scores_1_0_x_1_1 == pytest.approx(
|
||||
lexical_score_reference[1], rel=0.01
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_sparse_embedding_corner_case(server, pooling_task):
|
||||
if pooling_task != "token_classify":
|
||||
return
|
||||
|
||||
client = server.get_async_client()
|
||||
embeddings = await sparse_embeddings(client, ["Hi"])
|
||||
assert len(embeddings) == 1
|
||||
assert 2673 in embeddings[0]
|
||||
assert embeddings[0][2673] == pytest.approx(0.26710861921310425, rel=0.01)
|
||||
|
||||
|
||||
# https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L163
|
||||
def colbert_score(q_reps: torch.Tensor, p_reps: torch.Tensor) -> torch.Tensor:
|
||||
token_scores = torch.einsum("in,jn->ij", q_reps, p_reps)
|
||||
scores, _ = token_scores.max(-1)
|
||||
scores = torch.sum(scores) / q_reps.size(0)
|
||||
return scores
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bge_m3_api_server_multi_vector(server, pooling_task):
|
||||
client = server.get_async_client()
|
||||
|
||||
if pooling_task != "token_embed":
|
||||
with pytest.raises(openai.BadRequestError):
|
||||
await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
return
|
||||
|
||||
result_1 = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
embeddings_1 = [torch.tensor(data["data"]) for data in result_1.json()["data"]]
|
||||
|
||||
result_2 = await client.post(
|
||||
"../pooling",
|
||||
body={"model": MODEL_NAME, "input": sentences_2, "task": "token_embed"},
|
||||
cast_to=httpx.Response,
|
||||
)
|
||||
embeddings_2 = [torch.tensor(data["data"]) for data in result_2.json()["data"]]
|
||||
|
||||
colbert_score_1_0_x_2_0 = colbert_score(embeddings_1[0], embeddings_2[0])
|
||||
assert colbert_score_1_0_x_2_0 == pytest.approx(
|
||||
colbert_score_reference[0], rel=0.01
|
||||
)
|
||||
colbert_score_1_0_x_2_1 = colbert_score(embeddings_1[0], embeddings_2[1])
|
||||
assert colbert_score_1_0_x_2_1 == pytest.approx(
|
||||
colbert_score_reference[1], rel=0.01
|
||||
)
|
||||
@@ -1,108 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"jason9693/Qwen2.5-1.5B-apeach",
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"] if current_platform.is_rocm() else ["float"])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
# the tolerance value of 1e-2 is selected based on the
|
||||
# half datatype tests in
|
||||
# tests/models/language/pooling/test_embedding.py
|
||||
assert torch.allclose(
|
||||
hf_output,
|
||||
vllm_output,
|
||||
rtol=2e-3 if dtype == "float" else 1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_bert_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "cross-encoder/ms-marco-TinyBERT-L-2-v2"
|
||||
score_inputs = (
|
||||
"What is the capital of France?",
|
||||
[
|
||||
"Paris.",
|
||||
"Paris is the capital and largest city of France.",
|
||||
"William Shakespeare wrote Hamlet in the early seventeenth century.",
|
||||
],
|
||||
)
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(
|
||||
model, dtype="half", auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
# HfRunner uses problem_type to preserve the model's
|
||||
# sbert_ce_default_activation_function=Identity raw logits.
|
||||
hf_model.config.problem_type = "regression"
|
||||
hf_outputs = [hf_model.classify(prompts) for prompts in prompt_batches]
|
||||
|
||||
text_1, text_2 = score_inputs
|
||||
text_pairs = [[text_1, document] for document in text_2]
|
||||
with hf_runner(model, dtype="half", is_cross_encoder=True) as hf_model:
|
||||
hf_scores = hf_model.predict(text_pairs).tolist()
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.classify(prompts) for prompts in prompt_batches]
|
||||
vllm_scores = vllm_model.score(*score_inputs)
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
hf_tensor = torch.tensor(hf_batch)
|
||||
vllm_tensor = torch.tensor(vllm_batch)
|
||||
assert vllm_tensor.shape == hf_tensor.shape
|
||||
assert torch.allclose(vllm_tensor, hf_tensor, rtol=1e-2, atol=1e-4)
|
||||
|
||||
assert torch.allclose(
|
||||
torch.tensor(vllm_scores),
|
||||
torch.tensor(hf_scores),
|
||||
rtol=1e-2,
|
||||
atol=1e-4,
|
||||
)
|
||||
@@ -1,374 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColBERT late interaction scoring.
|
||||
|
||||
Tests are parametrized across multiple ColBERT backbones to ensure the
|
||||
generic ColBERT support works with different encoder architectures.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
|
||||
# -----------------------------------------------------------------------
|
||||
COLBERT_MODELS = {
|
||||
"bert": {
|
||||
"model": "answerdotai/answerai-colbert-small-v1",
|
||||
"colbert_dim": 96,
|
||||
"max_model_len": 512,
|
||||
"extra_kwargs": {},
|
||||
"hf_comparison": {
|
||||
"weights_file": "model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "BertModel",
|
||||
},
|
||||
},
|
||||
"modernbert": {
|
||||
"model": "lightonai/GTE-ModernColBERT-v1",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 299,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTModernBertModel"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "1_Dense/model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
"jina": {
|
||||
"model": "jinaai/jina-colbert-v2",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 8192,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTJinaRobertaModel"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": True,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
"lfm2": {
|
||||
"model": "LiquidAI/LFM2-ColBERT-350M",
|
||||
"colbert_dim": 128,
|
||||
"max_model_len": 511,
|
||||
"extra_kwargs": {
|
||||
"hf_overrides": {
|
||||
"architectures": ["ColBERTLfm2Model"],
|
||||
},
|
||||
},
|
||||
"hf_comparison": {
|
||||
"weights_file": "1_Dense/model.safetensors",
|
||||
"weights_key": "linear.weight",
|
||||
"trust_remote_code": False,
|
||||
"model_cls": "AutoModel",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
TEXTS_1 = [
|
||||
"What is the capital of France?",
|
||||
"What is the capital of Germany?",
|
||||
]
|
||||
|
||||
TEXTS_2 = [
|
||||
"The capital of France is Paris.",
|
||||
"The capital of Germany is Berlin.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load HF model on the given device with a compatible attention impl."""
|
||||
from transformers import AutoModel, BertModel
|
||||
|
||||
cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel
|
||||
trust = hf_spec.get("trust_remote_code", False)
|
||||
|
||||
# Flash / Triton kernels require GPU tensors; fall back to eager on CPU.
|
||||
extra = {}
|
||||
if device.type == "cpu":
|
||||
extra["attn_implementation"] = "eager"
|
||||
|
||||
model = cls.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=trust,
|
||||
**extra,
|
||||
).to(device)
|
||||
model.eval()
|
||||
|
||||
# Transformers 5.0 weight materialization can clear non-persistent
|
||||
# buffers (e.g. rotary inv_freq) that were registered with
|
||||
# persistent=False. Re-compute them so the model produces valid output.
|
||||
for mod in model.modules():
|
||||
if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
|
||||
mod.inv_freq = mod._compute_inv_freq(device=device)
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Download and return the ColBERT linear projection weight."""
|
||||
from safetensors.torch import load_file
|
||||
|
||||
from vllm.transformers_utils.repo_utils import hf_api, with_retry
|
||||
|
||||
path = with_retry(
|
||||
lambda: hf_api().hf_hub_download(model_name, filename=hf_spec["weights_file"]),
|
||||
f"Error downloading {hf_spec['weights_file']} from {model_name}",
|
||||
)
|
||||
weights = load_file(path)
|
||||
return weights[hf_spec["weights_key"]].to(device)
|
||||
|
||||
|
||||
def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device):
|
||||
"""Run HF model + projection and return L2-normalised token embeddings."""
|
||||
import torch.nn.functional as F
|
||||
|
||||
embeddings = []
|
||||
for text in texts:
|
||||
inputs = tokenizer(text, return_tensors="pt").to(device)
|
||||
with torch.no_grad():
|
||||
hidden = model(**inputs).last_hidden_state.float()
|
||||
projected = F.linear(hidden, linear_weight.float())
|
||||
normalised = F.normalize(projected, p=2, dim=-1)
|
||||
embeddings.append(normalised.squeeze(0).cpu())
|
||||
return embeddings
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
|
||||
"""Load the HF backbone + ColBERT projection, freeing the GPU on exit.
|
||||
|
||||
These live outside any runner context, so without explicit cleanup ROCm
|
||||
keeps the VRAM resident and the next backend parametrization (or test)
|
||||
OOMs on startup.
|
||||
"""
|
||||
hf_model = _load_hf_model(model_name, hf_spec, device)
|
||||
linear_weight = _load_projection_weight(model_name, hf_spec, device)
|
||||
try:
|
||||
yield hf_model, linear_weight
|
||||
finally:
|
||||
del hf_model, linear_weight
|
||||
cleanup_dist_env_and_memory()
|
||||
wait_for_memory_to_settle()
|
||||
|
||||
|
||||
def _assert_embeddings_close(vllm_outputs, hf_embeddings):
|
||||
"""Assert that vLLM and HuggingFace embeddings match."""
|
||||
for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
|
||||
vllm_emb = torch.as_tensor(vllm_out).float()
|
||||
|
||||
assert hf_emb.shape == vllm_emb.shape, (
|
||||
f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}"
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
vllm_emb,
|
||||
hf_emb,
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
msg=f"Embedding mismatch for text {i}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="class")
|
||||
def colbert_spec(request):
|
||||
"""Return the model spec dict for the current parametrization."""
|
||||
return COLBERT_MODELS[request.param]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_model_name(colbert_spec):
|
||||
return colbert_spec["model"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_dim(colbert_spec):
|
||||
return colbert_spec["colbert_dim"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_max_model_len(colbert_spec):
|
||||
return colbert_spec["max_model_len"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_extra_kwargs(colbert_spec):
|
||||
return colbert_spec["extra_kwargs"]
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def colbert_model(
|
||||
vllm_runner,
|
||||
colbert_model_name,
|
||||
colbert_max_model_len,
|
||||
colbert_extra_kwargs,
|
||||
):
|
||||
with vllm_runner(
|
||||
colbert_model_name,
|
||||
runner="pooling",
|
||||
dtype=DTYPE,
|
||||
max_model_len=colbert_max_model_len,
|
||||
enforce_eager=True,
|
||||
**colbert_extra_kwargs,
|
||||
) as vllm_model:
|
||||
yield vllm_model
|
||||
|
||||
|
||||
class TestColbertSharedEngine:
|
||||
"""Tests sharing one engine per model.
|
||||
|
||||
Class-scoped so the engine is released before `test_colbert_hf_comparison`,
|
||||
which needs a runner of its own and cannot start while this one holds VRAM.
|
||||
"""
|
||||
|
||||
def test_colbert_token_embed(self, colbert_model, colbert_dim):
|
||||
"""Test that ColBERT model produces token embeddings."""
|
||||
outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.as_tensor(outputs[0])
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == colbert_dim
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
def test_colbert_late_interaction_1_to_1(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with 1:1 query-document pair."""
|
||||
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
d_outputs = colbert_model.token_embed([TEXTS_2[0]])
|
||||
|
||||
q_emb = torch.as_tensor(q_outputs[0])
|
||||
d_emb = torch.as_tensor(d_outputs[0])
|
||||
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2[0])
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
def test_colbert_late_interaction_1_to_N(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with 1:N query-documents."""
|
||||
q_outputs = colbert_model.token_embed([TEXTS_1[0]])
|
||||
d_outputs = colbert_model.token_embed(TEXTS_2)
|
||||
|
||||
q_emb = torch.as_tensor(q_outputs[0])
|
||||
|
||||
manual_scores = []
|
||||
for d_out in d_outputs:
|
||||
d_emb = torch.as_tensor(d_out)
|
||||
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1[0], TEXTS_2)
|
||||
|
||||
assert len(vllm_scores) == 2
|
||||
for i in range(2):
|
||||
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
|
||||
|
||||
def test_colbert_late_interaction_N_to_N(self, colbert_model):
|
||||
"""Test ColBERT late interaction scoring with N:N query-documents."""
|
||||
q_outputs = colbert_model.token_embed(TEXTS_1)
|
||||
d_outputs = colbert_model.token_embed(TEXTS_2)
|
||||
|
||||
manual_scores = []
|
||||
for q_out, d_out in zip(q_outputs, d_outputs):
|
||||
q_emb = torch.as_tensor(q_out)
|
||||
d_emb = torch.as_tensor(d_out)
|
||||
manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
|
||||
|
||||
vllm_scores = colbert_model.score(TEXTS_1, TEXTS_2)
|
||||
|
||||
assert len(vllm_scores) == 2
|
||||
for i in range(2):
|
||||
assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
|
||||
|
||||
def test_colbert_relevance_ordering(self, colbert_model):
|
||||
"""Test that ColBERT scores relevant documents higher than irrelevant."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"Python is a programming language.",
|
||||
"Deep learning uses neural networks.",
|
||||
]
|
||||
|
||||
scores = colbert_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] > scores[1], "ML doc should score higher than Python doc"
|
||||
assert scores[2] > scores[1], "DL doc should score higher than Python doc"
|
||||
|
||||
def test_colbert_embed_not_supported(self, colbert_model):
|
||||
"""Test that ColBERT model does not support the embed task."""
|
||||
with pytest.raises(ValueError, match="Embedding API is not supported"):
|
||||
colbert_model.embed([TEXTS_1[0]])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "use_v2"),
|
||||
[
|
||||
pytest.param("bert", True, id="bert-v2"),
|
||||
pytest.param("modernbert", True, id="modernbert-v2"),
|
||||
pytest.param("jina", True, id="jina-v2"),
|
||||
pytest.param("lfm2", True, id="lfm2-v2"),
|
||||
],
|
||||
)
|
||||
def test_colbert_hf_comparison(vllm_runner, monkeypatch, backend, use_v2):
|
||||
"""Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
spec = COLBERT_MODELS[backend]
|
||||
hf_spec = spec["hf_comparison"]
|
||||
extra_kwargs = spec["extra_kwargs"]
|
||||
model_name = spec["model"]
|
||||
assert isinstance(model_name, str)
|
||||
assert isinstance(hf_spec, dict)
|
||||
assert isinstance(extra_kwargs, dict)
|
||||
test_texts = [TEXTS_1[0], TEXTS_2[0]]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2 else "0")
|
||||
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
runner="pooling",
|
||||
dtype="float32",
|
||||
max_model_len=spec["max_model_len"],
|
||||
enforce_eager=True,
|
||||
**extra_kwargs,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner == use_v2
|
||||
vllm_outputs = vllm_model.token_embed(test_texts)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
hf_tokenizer = AutoTokenizer.from_pretrained(
|
||||
model_name,
|
||||
trust_remote_code=hf_spec.get("trust_remote_code", False),
|
||||
)
|
||||
with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
|
||||
hf_embeddings = _compute_hf_colbert_embeddings(
|
||||
hf_model,
|
||||
hf_tokenizer,
|
||||
linear_weight,
|
||||
test_texts,
|
||||
device,
|
||||
)
|
||||
|
||||
_assert_embeddings_close(vllm_outputs, hf_embeddings)
|
||||
@@ -1,219 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Be careful of the order of models, decoder-only models should be
|
||||
# placed before encoder-only models, otherwise `Qwen2.5-0.5B-Instruct`
|
||||
# case won't pass because gte-Qwen2-1.5B-instruct will cache custom
|
||||
# model code with bidirectional attention.
|
||||
# [Decoder-only]
|
||||
pytest.param(
|
||||
"BAAI/bge-multilingual-gemma2",
|
||||
marks=[pytest.mark.core_model, pytest.mark.slow_test],
|
||||
),
|
||||
pytest.param(
|
||||
"intfloat/e5-mistral-7b-instruct",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"ssmits/Qwen2-7B-Instruct-embed-base", marks=[pytest.mark.cpu_model]
|
||||
),
|
||||
# [Encoder-only]
|
||||
pytest.param(
|
||||
"BAAI/bge-base-en-v1.5",
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
|
||||
pytest.param("intfloat/multilingual-e5-small"),
|
||||
# [Cross-Encoder]
|
||||
pytest.param(
|
||||
"sentence-transformers/stsb-roberta-base-v2",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model,
|
||||
) -> None:
|
||||
vllm_extra_kwargs = {}
|
||||
if model == "ssmits/Qwen2-7B-Instruct-embed-base":
|
||||
vllm_extra_kwargs["pooler_config"] = PoolerConfig(
|
||||
seq_pooling_type="MEAN", use_activation=False
|
||||
)
|
||||
|
||||
max_model_len: int | None = 512
|
||||
if model in [
|
||||
"sentence-transformers/all-MiniLM-L12-v2",
|
||||
"sentence-transformers/stsb-roberta-base-v2",
|
||||
]:
|
||||
max_model_len = None
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model, runner="pooling", max_model_len=max_model_len, **vllm_extra_kwargs
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"BAAI/bge-base-en-v1.5",
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_encoder_only_model_runner_v2_attention(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
) -> None:
|
||||
prompts = [
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for prompt in prompts:
|
||||
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
|
||||
output = hf_model.model(**hf_model.wrap_device(inputs))
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state[0, -1].float(), dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="float",
|
||||
max_model_len=64,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
pooler_config=PoolerConfig(
|
||||
task="embed", seq_pooling_type="LAST", use_activation=True
|
||||
),
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_encoder_model_runner_v2(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
model = "sentence-transformers/all-MiniLM-L6-v2"
|
||||
prompt_batches = [
|
||||
["short input"],
|
||||
[
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
],
|
||||
]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = [hf_model.encode(prompts) for prompts in prompt_batches]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = [vllm_model.embed(prompts) for prompts in prompt_batches]
|
||||
|
||||
for hf_batch, vllm_batch in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_batch,
|
||||
embeddings_1_lst=vllm_batch,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_matryoshka_dimensions_model_runner_v2(
|
||||
hf_runner, vllm_runner, monkeypatch
|
||||
) -> None:
|
||||
model = "Snowflake/snowflake-arctic-embed-m-v1.5"
|
||||
prompts = ["short input", "a longer input for a different output width"]
|
||||
dimensions = [None, 256]
|
||||
|
||||
with hf_runner(model, is_sentence_transformer=True) as hf_model:
|
||||
hf_outputs = hf_model.encode(prompts)
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
max_model_len=64,
|
||||
gpu_memory_utilization=0.25,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.embed(
|
||||
prompts,
|
||||
pooling_params=[PoolingParams(dimensions=d) for d in dimensions],
|
||||
)
|
||||
|
||||
expected_outputs = []
|
||||
for output, dimension in zip(hf_outputs, dimensions):
|
||||
output = torch.as_tensor(output)
|
||||
if dimension is not None:
|
||||
output = torch.nn.functional.normalize(output[:dimension], dim=0)
|
||||
expected_outputs.append(output.tolist())
|
||||
|
||||
assert [len(output) for output in vllm_outputs] == [768, 256]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=expected_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,48 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import TokensPrompt
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Qwen/Qwen3-0.6B"],
|
||||
)
|
||||
@torch.inference_mode
|
||||
def test_extract_hidden_states(hf_runner, vllm_runner, model: str):
|
||||
n_prompt_tokens = [55, 56, 57]
|
||||
token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
enable_prefix_caching=True,
|
||||
) as vllm_model:
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
|
||||
for n, output in zip(n_prompt_tokens, pooling_outputs):
|
||||
assert len(output.prompt_token_ids) == n
|
||||
assert len(output.outputs.data) == n
|
||||
assert output.num_cached_tokens == 0
|
||||
|
||||
# test enable_prefix_caching plus all pooling
|
||||
# we need to skip reading cache at this request by
|
||||
# request.skip_reading_prefix_cache
|
||||
pooling_outputs = vllm_model.llm.encode(
|
||||
[TokensPrompt(prompt_token_ids=t) for t in token_prompts],
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
|
||||
for n, output in zip(n_prompt_tokens, pooling_outputs):
|
||||
assert len(output.prompt_token_ids) == n
|
||||
assert len(output.outputs.data) == n
|
||||
assert output.num_cached_tokens == 0
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["nie3e/sentiment-polish-gpt2-small"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for head_dtype_str in ["float32", "model"]:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
hf_overrides={"head_dtype": head_dtype_str},
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
model_dtype = model_config.dtype
|
||||
head_dtype = model_config.head_dtype
|
||||
|
||||
if head_dtype_str == "float32":
|
||||
assert head_dtype == torch.float32
|
||||
elif head_dtype_str == "model":
|
||||
assert head_dtype == model_dtype
|
||||
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Backbone validation for Jina Embeddings V5.
|
||||
|
||||
The V5 family ships two backbones under one `architectures` entry: `-small` is
|
||||
a Qwen3 decoder, while `-nano` is a bidirectional EuroBERT encoder. Upstream
|
||||
ships a separate `configuration_*.py` per repository, so the only signal
|
||||
distinguishing them is `is_decoder`, which the encoder variant sets to False.
|
||||
`JinaEmbeddingsV5ModelConfig` uses it to enable bidirectional attention for the
|
||||
encoder variant; `JinaEmbeddingsV5Model` then dispatches to the correct backbone.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.config import (
|
||||
MODELS_CONFIG_MAP,
|
||||
JinaEmbeddingsV5ModelConfig,
|
||||
)
|
||||
|
||||
|
||||
def _model_config(hf_config: PreTrainedConfig) -> ModelConfig:
|
||||
"""Minimal stand-in for ModelConfig; only hf_config is read."""
|
||||
return cast(ModelConfig, SimpleNamespace(hf_config=hf_config))
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_registered_for_the_architecture():
|
||||
"""The handler only runs if it is wired to the architecture name."""
|
||||
assert MODELS_CONFIG_MAP["JinaEmbeddingsV5Model"] is JinaEmbeddingsV5ModelConfig
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_encoder_backbone_enables_bidirectional_attention():
|
||||
"""An encoder checkpoint (is_decoder=False) is supported.
|
||||
|
||||
The handler sets is_causal=False so the Llama backbone uses
|
||||
EncoderOnlyAttention; JinaEmbeddingsV5Model then dispatches to the encoder
|
||||
implementation.
|
||||
"""
|
||||
hf_config = PreTrainedConfig(is_decoder=False)
|
||||
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(hf_config))
|
||||
|
||||
assert hf_config.is_causal is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_supported_decoder_backbone_is_accepted():
|
||||
"""The Qwen3-based variants must keep loading.
|
||||
|
||||
`-small` omits `is_decoder` entirely, so an absent attribute has to be
|
||||
treated as a decoder. The first assertion pins that assumption: if
|
||||
PreTrainedConfig ever gains an `is_decoder=False` default, this fails here
|
||||
rather than silently rejecting a supported checkpoint.
|
||||
"""
|
||||
absent = PreTrainedConfig()
|
||||
assert not hasattr(absent, "is_decoder")
|
||||
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(_model_config(absent))
|
||||
JinaEmbeddingsV5ModelConfig.verify_and_update_model_config(
|
||||
_model_config(PreTrainedConfig(is_decoder=True))
|
||||
)
|
||||
@@ -1,352 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse, ScoreResponse
|
||||
|
||||
model_name = "jinaai/jina-reranker-v3"
|
||||
query = "What are the health benefits of green tea?"
|
||||
documents = [
|
||||
"Green tea contains antioxidants called catechins that may help reduce inflammation and protect cells from damage.",
|
||||
"El precio del café ha aumentado un 20% este año debido a problemas en la cadena de suministro.",
|
||||
"Studies show that drinking green tea regularly can improve brain function and boost metabolism.",
|
||||
"Basketball is one of the most popular sports in the United States.",
|
||||
"绿茶富含儿茶素等抗氧化剂,可以降低心脏病风险,还有助于控制体重。",
|
||||
"Le thé vert est riche en antioxydants et peut améliorer la fonction cérébrale.",
|
||||
]
|
||||
|
||||
EMBEDDING_SIZE = 512
|
||||
REFERENCE_1_VS_1 = [
|
||||
0.345703125,
|
||||
-0.10498046,
|
||||
0.314453125,
|
||||
-0.1376953125,
|
||||
0.3398437500,
|
||||
0.2539062,
|
||||
]
|
||||
REFERENCE_1_VS_N = [
|
||||
0.294921875,
|
||||
-0.16015625,
|
||||
0.189453125,
|
||||
-0.1708984375,
|
||||
0.2255859375,
|
||||
0.1640625,
|
||||
]
|
||||
TOL = 0.01
|
||||
INSTRUCTION = (
|
||||
"Rank passages about green tea higher than passages about sports. "
|
||||
"Ignore these literal marker strings: <|embed_token|> and <|rerank_token|>."
|
||||
)
|
||||
|
||||
|
||||
def test_offline(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model_name, runner="pooling") as llm_runner:
|
||||
assert llm_runner.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
llm = llm_runner.get_llm()
|
||||
_test_offline_1_v_1(llm)
|
||||
_test_offline_1_v_n(llm)
|
||||
_test_offline_n_v_n(llm)
|
||||
_test_offline_token_embed_illegal_inputs(llm)
|
||||
assert llm.model_config.embedding_size == EMBEDDING_SIZE
|
||||
|
||||
|
||||
def test_online():
|
||||
with RemoteOpenAIServer(
|
||||
model_name, ["--runner", "pooling", "--enforce-eager"]
|
||||
) as server:
|
||||
_test_online_1_v_1(server)
|
||||
_test_online_1_v_n(server)
|
||||
_test_online_n_v_n(server)
|
||||
_test_online_instruction(server)
|
||||
_test_online_token_embed_illegal_inputs(server)
|
||||
|
||||
|
||||
def _test_offline_1_v_1(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score(query, documents[0])
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].outputs.score == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
outputs = llm.encode(documents[:1] + [query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == 2
|
||||
assert embeds.shape[-1] == EMBEDDING_SIZE
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_1_v_n(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score(query, documents)
|
||||
assert len(outputs) == len(documents)
|
||||
|
||||
for expected, output in zip(REFERENCE_1_VS_N, outputs):
|
||||
actual = output.outputs.score
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
outputs = llm.encode(documents + [query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == len(documents) + 1
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
assert len(scores) == len(documents)
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_n_v_n(llm):
|
||||
# test llm.score
|
||||
outputs = llm.score([query] * len(documents), documents)
|
||||
assert len(outputs) == len(documents)
|
||||
|
||||
for expected, output in zip(REFERENCE_1_VS_1, outputs):
|
||||
actual = output.outputs.score
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test llm.encode
|
||||
for doc, expected in zip(documents, REFERENCE_1_VS_1):
|
||||
outputs = llm.encode([doc, query], pooling_task="token_embed")
|
||||
embeds = outputs[0].outputs.data.float()
|
||||
assert embeds.shape[0] == 2
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_offline_token_embed_illegal_inputs(llm):
|
||||
with pytest.raises(
|
||||
ValueError, match="The JinaForRanking model requires at least 2 inputs."
|
||||
):
|
||||
llm.encode([query], pooling_task="token_embed")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="The JinaForRanking model only supports text as input."
|
||||
):
|
||||
llm.encode([1, 2, 3], pooling_task="token_embed")
|
||||
|
||||
|
||||
def _get_score_response(server, query, document, **extra_body):
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"queries": query,
|
||||
"documents": document,
|
||||
}
|
||||
payload.update(extra_body)
|
||||
score_response = requests.post(
|
||||
server.url_for("score"),
|
||||
json=payload,
|
||||
)
|
||||
|
||||
score_response.raise_for_status()
|
||||
return ScoreResponse.model_validate(score_response.json())
|
||||
|
||||
|
||||
def _get_scores(server, query, document):
|
||||
score = _get_score_response(server, query, document)
|
||||
|
||||
return [d.score for d in score.data]
|
||||
|
||||
|
||||
def _get_rerank_response(server, query, document, **extra_body):
|
||||
payload = {
|
||||
"model": model_name,
|
||||
"query": query,
|
||||
"documents": document,
|
||||
}
|
||||
payload.update(extra_body)
|
||||
rerank_response = requests.post(
|
||||
server.url_for("rerank"),
|
||||
json=payload,
|
||||
)
|
||||
|
||||
rerank_response.raise_for_status()
|
||||
return RerankResponse.model_validate(rerank_response.json())
|
||||
|
||||
|
||||
def _get_embeds(server, prompts: list[str]):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": prompts,
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
poolings = PoolingResponse.model_validate(response.json())
|
||||
|
||||
return torch.as_tensor([d.data for d in poolings.data][0]).float()
|
||||
|
||||
|
||||
def _test_online_1_v_1(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, query, documents[0])
|
||||
assert len(scores) == 1
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
embeds = _get_embeds(server, [documents[0], query])
|
||||
assert embeds.shape[0] == 2
|
||||
assert embeds.shape[-1] == EMBEDDING_SIZE
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
|
||||
|
||||
|
||||
def _test_online_1_v_n(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, query, documents)
|
||||
assert len(scores) == len(documents)
|
||||
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
embeds = _get_embeds(server, documents + [query])
|
||||
assert embeds.shape[0] == len(documents) + 1
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
assert len(scores) == len(documents)
|
||||
for expected, actual in zip(REFERENCE_1_VS_N, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_online_n_v_n(server):
|
||||
# test scoring api
|
||||
scores = _get_scores(server, [query] * len(documents), documents)
|
||||
assert len(scores) == len(documents)
|
||||
|
||||
for expected, actual in zip(REFERENCE_1_VS_1, scores):
|
||||
assert actual == pytest.approx(expected, abs=TOL)
|
||||
|
||||
# test pooling api
|
||||
for doc, expected in zip(documents, REFERENCE_1_VS_1):
|
||||
embeds = _get_embeds(server, [doc, query])
|
||||
assert embeds.shape[0] == 2
|
||||
|
||||
doc_embeds = embeds[:-1]
|
||||
query_embeds = embeds[-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
assert len(scores) == 1
|
||||
assert scores[0] == pytest.approx(expected, abs=TOL)
|
||||
|
||||
|
||||
def _test_online_instruction(server):
|
||||
docs = documents[:2]
|
||||
|
||||
default_score = _get_score_response(server, query, docs)
|
||||
instruction_score = _get_score_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
instruction=INSTRUCTION,
|
||||
)
|
||||
kwargs_score = _get_score_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
chat_template_kwargs={"instruction": INSTRUCTION},
|
||||
)
|
||||
|
||||
assert instruction_score.usage.prompt_tokens > default_score.usage.prompt_tokens
|
||||
assert kwargs_score.usage.prompt_tokens == instruction_score.usage.prompt_tokens
|
||||
assert len(instruction_score.data) == len(default_score.data)
|
||||
assert [d.score for d in kwargs_score.data] == pytest.approx(
|
||||
[d.score for d in instruction_score.data], abs=TOL
|
||||
)
|
||||
|
||||
default_rerank = _get_rerank_response(server, query, docs)
|
||||
instruction_rerank = _get_rerank_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
instruction=INSTRUCTION,
|
||||
)
|
||||
kwargs_rerank = _get_rerank_response(
|
||||
server,
|
||||
query,
|
||||
docs,
|
||||
chat_template_kwargs={"instruction": INSTRUCTION},
|
||||
)
|
||||
|
||||
assert instruction_rerank.usage.prompt_tokens > default_rerank.usage.prompt_tokens
|
||||
assert kwargs_rerank.usage.prompt_tokens == instruction_rerank.usage.prompt_tokens
|
||||
assert len(instruction_rerank.results) == len(default_rerank.results)
|
||||
assert [r.relevance_score for r in kwargs_rerank.results] == pytest.approx(
|
||||
[r.relevance_score for r in instruction_rerank.results], abs=TOL
|
||||
)
|
||||
|
||||
|
||||
def _test_online_token_embed_illegal_inputs(server):
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": [query],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking model requires at least 2 inputs."
|
||||
)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"input": [1, 2, 3],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking model only supports text as input."
|
||||
)
|
||||
|
||||
response = requests.post(
|
||||
server.url_for("pooling"),
|
||||
json={
|
||||
"model": model_name,
|
||||
"task": "token_embed",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "The cat sat on the mat.",
|
||||
}
|
||||
],
|
||||
"encoding_format": "float",
|
||||
},
|
||||
)
|
||||
assert response.json()["error"]["message"].startswith(
|
||||
"The JinaForRanking does not support chat Request."
|
||||
)
|
||||
@@ -1,227 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for max_tokens_per_doc and max_tokens_per_query."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
|
||||
|
||||
TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
|
||||
ExpectedPromptTokens = int | tuple[int, ...]
|
||||
|
||||
long_query = "What is the capital of France?" * 20
|
||||
long_doc = "The capital of France is Paris. " * 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestConfig:
|
||||
model: str
|
||||
args: list[str]
|
||||
without_truncated_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_query_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_doc_prompt_tokens: ExpectedPromptTokens
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens: ExpectedPromptTokens
|
||||
|
||||
|
||||
RERANK_CONFIGS = [
|
||||
# 1. cross-encoder
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v2-base-multilingual",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=284,
|
||||
with_max_tokens_per_query_prompt_tokens=154,
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 2. cross-encoder + score template
|
||||
TestConfig(
|
||||
model="Qwen/Qwen3-Reranker-0.6B",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--hf-overrides",
|
||||
json.dumps(
|
||||
{
|
||||
"architectures": ["Qwen3ForSequenceClassification"],
|
||||
"classifier_from_token": ["no", "yes"],
|
||||
"is_original_qwen3_reranker": True,
|
||||
}
|
||||
),
|
||||
"--chat-template",
|
||||
os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
|
||||
],
|
||||
without_truncated_prompt_tokens=352,
|
||||
with_max_tokens_per_query_prompt_tokens=223,
|
||||
with_max_tokens_per_doc_prompt_tokens=221,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=92,
|
||||
),
|
||||
# 3. bi-encoder
|
||||
TestConfig(
|
||||
model="intfloat/multilingual-e5-small",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
# This model has produced both prompt-token totals in CI/local cache;
|
||||
# keep truncation checks exact while tolerating the boundary delta.
|
||||
without_truncated_prompt_tokens=(284, 285),
|
||||
with_max_tokens_per_query_prompt_tokens=(154, 155),
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 4. late-interaction
|
||||
TestConfig(
|
||||
model="answerdotai/answerai-colbert-small-v1",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=284,
|
||||
with_max_tokens_per_query_prompt_tokens=154,
|
||||
with_max_tokens_per_doc_prompt_tokens=154,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=24,
|
||||
),
|
||||
# 5. jinaai/jina-reranker-v3
|
||||
TestConfig(
|
||||
model="jinaai/jina-reranker-v3",
|
||||
args=[
|
||||
"--enforce-eager",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--trust-remote-code",
|
||||
],
|
||||
without_truncated_prompt_tokens=567,
|
||||
with_max_tokens_per_query_prompt_tokens=308,
|
||||
with_max_tokens_per_doc_prompt_tokens=436,
|
||||
with_max_tokens_per_query_and_doc_prompt_tokens=177,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def assert_prompt_tokens(actual: int, expected: ExpectedPromptTokens) -> None:
|
||||
if isinstance(expected, int):
|
||||
assert actual == expected
|
||||
else:
|
||||
assert actual in expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
|
||||
def server(request):
|
||||
config: TestConfig = request.param
|
||||
env_dict = {"VLLM_LOGGING_LEVEL": "WARNING"}
|
||||
with RemoteOpenAIServer(
|
||||
config.model, config.args, env_dict=env_dict
|
||||
) as remote_server:
|
||||
yield config, remote_server
|
||||
|
||||
|
||||
def test_without_truncated(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={"model": config.model, "query": long_query, "documents": [long_doc]},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.without_truncated_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_query(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_query_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_doc_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def test_max_tokens_per_query_and_doc(server):
|
||||
"""Test that max_tokens_per_doc truncates documents correctly."""
|
||||
config, remote_server = server
|
||||
|
||||
response = requests.post(
|
||||
remote_server.url_for("rerank"),
|
||||
json={
|
||||
"model": config.model,
|
||||
"query": long_query,
|
||||
"documents": [long_doc],
|
||||
"max_tokens_per_query": 10,
|
||||
"max_tokens_per_doc": 10,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
rerank = RerankResponse.model_validate(response.json())
|
||||
|
||||
assert rerank.id is not None
|
||||
assert rerank.results is not None
|
||||
assert len(rerank.results) == 1
|
||||
assert_prompt_tokens(
|
||||
rerank.usage.prompt_tokens,
|
||||
config.with_max_tokens_per_query_and_doc_prompt_tokens,
|
||||
)
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
|
||||
|
||||
def test_idefics_multimodal(
|
||||
vllm_runner,
|
||||
) -> None:
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model_name="HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
runner="pooling",
|
||||
convert="classify",
|
||||
load_format="dummy",
|
||||
max_model_len=512,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=1,
|
||||
disable_log_stats=True,
|
||||
dtype="bfloat16",
|
||||
) as vllm_model:
|
||||
llm = vllm_model.get_llm()
|
||||
outputs = llm.classify(prompts)
|
||||
for output in outputs:
|
||||
assert len(output.outputs.probs) == 2
|
||||
|
||||
|
||||
def update_config(config):
|
||||
text_config = config.get_text_config()
|
||||
text_config.update(
|
||||
{
|
||||
"architectures": ["Gemma3ForSequenceClassification"],
|
||||
"classifier_from_token": ["A", "B", "C", "D", "E"],
|
||||
"method": "no_post_processing",
|
||||
"id2label": {
|
||||
"A": "Chair",
|
||||
"B": "Couch",
|
||||
"C": "Table",
|
||||
"D": "Bed",
|
||||
"E": "Cupboard",
|
||||
},
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def test_gemma_multimodal(
|
||||
vllm_runner,
|
||||
) -> None:
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": """
|
||||
You are a helpful assistant. You will be given a product description
|
||||
which may also include an image. Classify the following product into
|
||||
one of the categories:
|
||||
|
||||
A = chair
|
||||
B = couch
|
||||
C = table
|
||||
D = bed
|
||||
E = cupboard
|
||||
|
||||
You'll answer with exactly one letter (A, B, C, D, or E).""",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/red_chair.jpg"
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "A fine 19th century piece of furniture."},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model_name="google/gemma-3-4b-it",
|
||||
runner="pooling",
|
||||
convert="classify",
|
||||
load_format="auto",
|
||||
hf_overrides=update_config,
|
||||
pooler_config=PoolerConfig(seq_pooling_type="LAST"),
|
||||
max_model_len=512,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=1,
|
||||
disable_log_stats=True,
|
||||
dtype="bfloat16",
|
||||
) as vllm_model:
|
||||
llm = vllm_model.get_llm()
|
||||
prompts = llm._preprocess_chat([messages])
|
||||
|
||||
result = llm.classify(prompts)
|
||||
assert result[0].outputs.probs[0] > 0.95
|
||||
assert all(c < 0.05 for c in result[0].outputs.probs[1:])
|
||||
@@ -1,44 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_embeddings_close
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["BAAI/bge-m3"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_model_len=None,
|
||||
dtype=dtype,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(example_prompts)
|
||||
|
||||
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
embedding = output.last_hidden_state[0].float()
|
||||
hf_outputs.append(embedding.cpu())
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_output,
|
||||
embeddings_1_lst=vllm_output,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,34 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["Rami/multi-label-class-classification-on-github-issues"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output)
|
||||
vllm_output = torch.tensor(vllm_output)
|
||||
|
||||
assert torch.allclose(
|
||||
hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: SIM117
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from ...utils import EmbedModelInfo
|
||||
|
||||
MODELS = [
|
||||
EmbedModelInfo(
|
||||
"nomic-ai/nomic-embed-text-v1",
|
||||
# Fixme:
|
||||
# Update nomic-embed code to support the latest
|
||||
# HF version and remove revision set.
|
||||
revision="720244025c1a7e15661a174c63cce63c8218e52b",
|
||||
),
|
||||
# EmbedModelInfo("nomic-ai/nomic-embed-text-v1.5"),
|
||||
# EmbedModelInfo("nomic-ai/CodeRankEmbed"),
|
||||
EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe"),
|
||||
# EmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long"),
|
||||
]
|
||||
|
||||
rope_theta = 1000
|
||||
factor = 4.0
|
||||
original_max_position_embeddings = 2048
|
||||
max_model_len = int(original_max_position_embeddings * factor)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def predownload_models():
|
||||
"""Warm the HF cache so engine startup never hits transient hub errors."""
|
||||
from vllm.transformers_utils.repo_utils import hf_api, with_retry
|
||||
|
||||
for model_info in MODELS:
|
||||
with_retry(
|
||||
partial(
|
||||
hf_api().snapshot_download,
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
),
|
||||
f"Error downloading {model_info.name}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_default(model_info, vllm_runner):
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
|
||||
# For nomic-embed-text-v2-moe the length is set to 512
|
||||
# by sentence_bert_config.json.
|
||||
assert model_config.max_model_len == 512
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v1":
|
||||
assert model_config.max_model_len == 8192
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_set_max_model_len_legal(model_info, vllm_runner):
|
||||
# set max_model_len <= 512
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=256,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 256
|
||||
|
||||
# For nomic-embed-text-v2-moe the length is set to 512
|
||||
# by sentence_bert_config.json.
|
||||
if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
|
||||
with pytest.raises(ValueError):
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=1024,
|
||||
):
|
||||
pass
|
||||
return
|
||||
|
||||
# set 512 < max_model_len <= 2048
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=1024,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 1024
|
||||
|
||||
# set max_model_len > 2048
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=4096,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.max_model_len == 4096
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_use_rope_scaling_legal(model_info, vllm_runner):
|
||||
hf_overrides = {
|
||||
"rope_parameters": {
|
||||
"rope_theta": rope_theta,
|
||||
"rope_type": "yarn",
|
||||
"factor": factor,
|
||||
"original_max_position_embeddings": original_max_position_embeddings,
|
||||
},
|
||||
"max_model_len": max_model_len,
|
||||
}
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name,
|
||||
revision=model_info.revision,
|
||||
runner="pooling",
|
||||
max_model_len=None,
|
||||
hf_overrides=hf_overrides,
|
||||
):
|
||||
pass
|
||||
@@ -1,167 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.models.utils import softmax
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
["jason9693/Qwen2.5-1.5B-apeach", "papluca/xlm-roberta-base-language-detection"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_classify_models_using_activation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_activation_out = vllm_model.classify(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_activation_out = vllm_model.classify(example_prompts)
|
||||
|
||||
for wo_activation, w_activation in zip(wo_activation_out, w_activation_out):
|
||||
wo_activation = torch.tensor(wo_activation)
|
||||
w_activation = torch.tensor(w_activation)
|
||||
|
||||
assert not torch.allclose(wo_activation, w_activation, atol=1e-2), (
|
||||
"pooler_config is not working"
|
||||
)
|
||||
assert torch.allclose(
|
||||
softmax(wo_activation), w_activation, 1e-3 if dtype == "float" else 1e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_embed_models_using_normalize(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_normalize = torch.tensor(vllm_model.embed(example_prompts))
|
||||
|
||||
assert not torch.allclose(wo_normalize, w_normalize, atol=1e-2), (
|
||||
"pooler_config normalize is not working"
|
||||
)
|
||||
assert torch.allclose(
|
||||
F.normalize(wo_normalize, p=2, dim=-1), w_normalize, atol=1e-2
|
||||
), "w_normal should be close to normal(wo_normal)."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"internlm/internlm2-1_8b-reward",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_reward_models_using_activation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False),
|
||||
) as vllm_model:
|
||||
wo_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True),
|
||||
) as vllm_model:
|
||||
w_activation = vllm_model.token_classify(example_prompts)
|
||||
|
||||
for wo, w in zip(wo_activation, w_activation):
|
||||
wo = torch.tensor(wo)
|
||||
w = torch.tensor(w)
|
||||
|
||||
assert not torch.allclose(wo, w, atol=1e-2), (
|
||||
"pooler_config activation is not working"
|
||||
)
|
||||
assert torch.allclose(softmax(wo), w, atol=1e-2), (
|
||||
"w_activation should be close to activation(wo_activation)."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_multi_vector_retrieval_models_using_normalize(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=False, task="token_embed"),
|
||||
) as vllm_model:
|
||||
wo_normalize = vllm_model.token_embed(example_prompts)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=512,
|
||||
dtype=dtype,
|
||||
pooler_config=PoolerConfig(use_activation=True, task="token_embed"),
|
||||
) as vllm_model:
|
||||
w_normalize = vllm_model.token_embed(example_prompts)
|
||||
|
||||
for wo, w in zip(wo_normalize, w_normalize):
|
||||
assert not torch.allclose(wo, w, atol=1e-2), (
|
||||
"pooler_config normalize is not working"
|
||||
)
|
||||
assert torch.allclose(F.normalize(wo, p=2, dim=-1), w, atol=1e-2), (
|
||||
"w_normal should be close to normal(wo_normal)."
|
||||
)
|
||||
@@ -1,179 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
|
||||
from ....conftest import HfRunner
|
||||
from ....utils import VLLM_PATH
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import StrPath
|
||||
|
||||
|
||||
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
|
||||
assert FIXTURES_PATH.exists()
|
||||
FIXTURE_REWARD_RESULT = {
|
||||
"Qwen/Qwen2.5-Math-PRM-7B": FIXTURES_PATH / "qwen2_5_math_prm_reward_step.json",
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def math_step_prompts():
|
||||
# ruff: noqa: E501
|
||||
data = {
|
||||
"system": "Please reason step by step, and put your final answer within \\boxed{}. ",
|
||||
"query": "Sue lives in a fun neighborhood. One weekend, the neighbors decided to play a prank on Sue. On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard. On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and put these newly painted white flamingos back out on Sue's front yard. Then, on Sunday morning, they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more pink plastic flamingos were out than white plastic flamingos?",
|
||||
"response": [
|
||||
"To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink plastic flamingos.",
|
||||
"On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos and 6 white flamingos.",
|
||||
"On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.",
|
||||
"To find the difference, subtract the number of white flamingos from the number of pink flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out than white plastic flamingos. The answer is (\\boxed{30}).",
|
||||
],
|
||||
}
|
||||
answer = "<extra_0>".join(data["response"]) + "<extra_0>"
|
||||
prompt = f"<im_start>system\n{data['system']}<im_end>\n<im_start>user\n{data['query']}<im_end>\n<im_start>assistant\n{answer}<im_end><|endoftext|>"
|
||||
return [prompt]
|
||||
|
||||
|
||||
def step_reward_patch_hf_model(hf_model: HfRunner):
|
||||
# Patch the hf_runner to use the step reward function
|
||||
def make_step_rewards(
|
||||
logits: torch.Tensor, token_masks: torch.Tensor
|
||||
) -> list[list[float]]:
|
||||
probabilities = F.softmax(logits, dim=-1)
|
||||
probabilities = probabilities * token_masks.unsqueeze(-1)
|
||||
|
||||
all_scores_res: list[list[float]] = []
|
||||
for i in range(probabilities.size(0)):
|
||||
sample = probabilities[i] # seq_len, num_labels
|
||||
positive_probs = sample[sample != 0].view(-1, 2)
|
||||
non_zero_elements_list = positive_probs.cpu().tolist()
|
||||
all_scores_res.append(non_zero_elements_list)
|
||||
return all_scores_res
|
||||
|
||||
def reward(prompts: list[str]) -> list[list[float]]:
|
||||
input_ids = hf_model.tokenizer(prompts, return_tensors="pt").input_ids
|
||||
input_ids = hf_model.wrap_device(input_ids)
|
||||
outputs = hf_model.model(input_ids=input_ids)
|
||||
|
||||
step_sep_id = hf_model.tokenizer.encode("<extra_0>")[0]
|
||||
token_masks = input_ids == step_sep_id
|
||||
return make_step_rewards(outputs[0], token_masks)
|
||||
|
||||
hf_model.reward = reward # type: ignore[attr-defined]
|
||||
|
||||
return hf_model
|
||||
|
||||
|
||||
def dump_reward_outputs(outputs: list[list[float]], filename: "StrPath"):
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
json.dump(outputs, f)
|
||||
|
||||
|
||||
def load_reward_outputs(filename: "StrPath") -> list[list[float]]:
|
||||
with open(filename, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-Math-PRM-7B",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_prm_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
if current_platform.is_cpu():
|
||||
pytest.skip("CPU only supports V1")
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
hf_model = step_reward_patch_hf_model(hf_model)
|
||||
hf_outputs = hf_model.reward(math_step_prompts)
|
||||
|
||||
dump_reward_outputs(
|
||||
hf_outputs,
|
||||
FIXTURE_REWARD_RESULT[model],
|
||||
)
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = torch.tensor(hf_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(hf_output, vllm_output, 1.5e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-Math-PRM-7B",
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_prm_models_with_golden_outputs(
|
||||
vllm_runner,
|
||||
math_step_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
if not FIXTURE_REWARD_RESULT.get(model):
|
||||
pytest.skip(f"No available golden outputs for {model}.")
|
||||
|
||||
use_v2_model_runner = not current_platform.is_cpu()
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if use_v2_model_runner else "0")
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
dtype=dtype,
|
||||
kv_cache_memory_bytes=64 * MiB_bytes,
|
||||
) as vllm_model:
|
||||
assert (
|
||||
vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
vllm_outputs = vllm_model.token_classify(math_step_prompts)
|
||||
|
||||
golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
|
||||
|
||||
# check logits difference
|
||||
for golden_output, vllm_output in zip(golden_outputs, vllm_outputs):
|
||||
golden_output = torch.tensor(golden_output).float()
|
||||
vllm_output = torch.tensor(vllm_output).float()
|
||||
|
||||
assert torch.allclose(golden_output, vllm_output, 1.5e-2)
|
||||
@@ -1,281 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.models.bert import (
|
||||
BertMLMHead,
|
||||
SPLADESparsePooler,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Functional test: SPLADE formula correctness (no HF download needed)
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("B,T,H,V", [(2, 3, 5, 7)])
|
||||
@torch.inference_mode
|
||||
def test_splade_pooler_matches_reference_formula(B, T, H, V):
|
||||
"""Ensure SPLADESparsePooler forward() matches the mathematical formula:
|
||||
log1p(relu(logits)) -> max over sequence length (after masking)."""
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Prepare [B] sequences of shape [T, H]
|
||||
hs_list = [torch.randn(T, H) for _ in range(B)]
|
||||
hs_tenser = torch.cat(hs_list)
|
||||
|
||||
# Simulate PoolingMetadata (only required fields)
|
||||
prompt_lens = [T, T - 1]
|
||||
prompt_lens_tenser = torch.tensor(prompt_lens, dtype=torch.int32)
|
||||
token_ids = torch.tensor(
|
||||
[
|
||||
[101, 5, 102], # Batch 0: [CLS], token, [SEP]
|
||||
[101, 6, 6], # Batch 1: [CLS], token, token (last token ignored)
|
||||
],
|
||||
dtype=torch.long,
|
||||
)
|
||||
meta = PoolingMetadata(
|
||||
prompt_lens=prompt_lens_tenser,
|
||||
prompt_token_ids=token_ids,
|
||||
prompt_token_ids_cpu=token_ids,
|
||||
pooling_params=[PoolingParams(task="embed")] * B,
|
||||
pooling_states=[PoolingStates() for _ in range(B)],
|
||||
)
|
||||
|
||||
# MLM head (prefer BertMLMHead, fallback to Linear if unavailable)
|
||||
try:
|
||||
mlm_head = BertMLMHead(hidden_size=H, vocab_size=V, layer_norm_eps=1e-12)
|
||||
except Exception:
|
||||
mlm_head = nn.Linear(H, V, bias=True)
|
||||
|
||||
# Forward pass through SPLADE pooler
|
||||
pooler = SPLADESparsePooler(mlm_head=mlm_head, pooling="max", remove_cls_sep=True)
|
||||
pooled = pooler(hidden_states=hs_tenser, pooling_metadata=meta) # list of [V]
|
||||
|
||||
# Basic output checks
|
||||
assert isinstance(pooled, torch.Tensor) and len(pooled) == B
|
||||
for vec in pooled:
|
||||
assert vec.shape == (V,)
|
||||
assert torch.isfinite(vec).all()
|
||||
assert (vec >= 0).all(), "SPLADE outputs must be non-negative."
|
||||
|
||||
# Reference implementation for comparison
|
||||
def ref_one(hs: torch.Tensor, L: int, tid_row: torch.Tensor) -> torch.Tensor:
|
||||
keep = torch.ones(L, dtype=torch.bool)
|
||||
if L > 0 and tid_row[0].item() == 101: # remove CLS
|
||||
keep[0] = False
|
||||
if L > 0 and tid_row[L - 1].item() == 102: # remove SEP
|
||||
keep[L - 1] = False
|
||||
|
||||
valid = hs[:L][keep[:L]]
|
||||
if valid.numel() == 0:
|
||||
return torch.zeros(V, dtype=torch.float32)
|
||||
|
||||
logits = mlm_head(valid) # [L', V]
|
||||
scores = torch.log1p(torch.relu(logits)) # [L', V]
|
||||
return scores.max(dim=0).values.to(torch.float32)
|
||||
|
||||
torch.testing.assert_close(
|
||||
pooled[0],
|
||||
ref_one(hs_list[0], prompt_lens[0], token_ids[0]),
|
||||
rtol=1e-4,
|
||||
atol=1e-4,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
pooled[1],
|
||||
ref_one(hs_list[1], prompt_lens[1], token_ids[1]),
|
||||
rtol=1e-4,
|
||||
atol=1e-4,
|
||||
)
|
||||
|
||||
|
||||
def test_pooling_runner_gathers_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
pooling_params = PoolingParams(task="embed", requires_token_ids=True)
|
||||
runner.pooling_params = {1: pooling_params, 3: pooling_params}
|
||||
runner.pooling_states = {1: PoolingStates(), 3: PoolingStates()}
|
||||
runner.prompt_token_ids = {
|
||||
1: torch.tensor([101, 102]),
|
||||
3: torch.tensor([101, 11, 102]),
|
||||
}
|
||||
|
||||
input_batch = MagicMock(spec=InputBatch)
|
||||
input_batch.idx_mapping_np = np.array([3, 1], dtype=np.int32)
|
||||
input_batch.num_reqs = 2
|
||||
req_states = MagicMock(spec=RequestState)
|
||||
req_states.prompt_len = MagicMock(np=np.array([0, 2, 0, 3], dtype=np.int32))
|
||||
metadata = runner._get_pooling_metadata(
|
||||
input_batch, req_states, torch.device("cpu")
|
||||
)
|
||||
|
||||
expected = torch.tensor([[101, 11, 102], [101, 102, 0]])
|
||||
assert metadata.prompt_token_ids_cpu is not None
|
||||
assert metadata.prompt_token_ids is None
|
||||
assert metadata.prompt_token_ids_cpu.is_pinned() is False
|
||||
torch.testing.assert_close(
|
||||
metadata.prompt_lens, torch.tensor([3, 2], dtype=torch.int32)
|
||||
)
|
||||
torch.testing.assert_close(metadata.prompt_token_ids_cpu, expected)
|
||||
|
||||
|
||||
def test_pooling_runner_stores_only_required_token_ids() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.model = MagicMock()
|
||||
runner.supported_tasks = frozenset({"embed"})
|
||||
runner.pooling_params = {}
|
||||
runner.pooling_states = {}
|
||||
runner.prompt_token_ids = {}
|
||||
runner.late_interaction_runner = MagicMock()
|
||||
|
||||
runner.add_request("req-1", 1, PoolingParams(task="embed"), [101, 102])
|
||||
runner.add_request(
|
||||
"req-2",
|
||||
2,
|
||||
PoolingParams(task="embed", requires_token_ids=True),
|
||||
[101, 11, 102],
|
||||
)
|
||||
|
||||
assert 1 not in runner.prompt_token_ids
|
||||
torch.testing.assert_close(runner.prompt_token_ids[2], torch.tensor([101, 11, 102]))
|
||||
|
||||
|
||||
def test_pooling_runner_releases_aborted_late_interaction_doc() -> None:
|
||||
runner = PoolingRunner.__new__(PoolingRunner)
|
||||
runner.late_interaction_runner = LateInteractionRunner()
|
||||
|
||||
query_key = "query-abort"
|
||||
late_interaction_runner = runner.late_interaction_runner
|
||||
late_interaction_runner._query_cache[query_key] = torch.ones(2, 4)
|
||||
late_interaction_runner._query_uses[query_key] = 1
|
||||
late_interaction_runner._doc_query_keys["doc-req"] = query_key
|
||||
|
||||
runner.on_requests_finished({"doc-req"})
|
||||
|
||||
assert not late_interaction_runner._query_cache
|
||||
assert not late_interaction_runner._query_uses
|
||||
assert not late_interaction_runner._doc_query_keys
|
||||
|
||||
|
||||
def test_encoder_cache_reset_clears_late_interaction_state() -> None:
|
||||
# Cached query embeddings are only invalidated by weight reloads, which
|
||||
# reach the runner via reset_encoder_cache. Resetting the multi-modal
|
||||
# cache is unrelated and must leave them intact.
|
||||
runner = GPUModelRunner.__new__(GPUModelRunner)
|
||||
runner.encoder_cache = MagicMock()
|
||||
runner.pooling_runner = MagicMock()
|
||||
|
||||
runner.reset_mm_cache()
|
||||
|
||||
runner.encoder_cache.reset_mm_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_not_called()
|
||||
|
||||
runner.reset_encoder_cache()
|
||||
|
||||
runner.encoder_cache.reset_encoder_cache.assert_called_once_with()
|
||||
runner.pooling_runner.clear.assert_called_once_with()
|
||||
|
||||
|
||||
def test_pooling_runner_rejects_unsupported_selected_task() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "plugin"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "plugin"
|
||||
|
||||
with (
|
||||
patch.object(PoolingRunner, "get_supported_tasks", return_value=["embed"]),
|
||||
pytest.raises(ValueError, match="selects 'plugin'"),
|
||||
):
|
||||
PoolingRunner(model, vllm_config)
|
||||
|
||||
|
||||
def test_pooling_runner_supports_encoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_encoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "token_classify"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_keeps_decoder_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed", "token_classify"}
|
||||
|
||||
|
||||
def test_pooling_runner_keeps_decoder_token_embedding() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "token_embed"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
vllm_config.model_config.attn_type = "decoder"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed"
|
||||
|
||||
runner = PoolingRunner(model, vllm_config)
|
||||
|
||||
assert runner.supported_tasks == {"embed", "token_embed"}
|
||||
|
||||
|
||||
def test_pooling_runner_supports_embed_token_classification() -> None:
|
||||
model = MagicMock()
|
||||
model.pooler.get_supported_tasks.return_value = {"embed", "embed&token_classify"}
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_seqs = 2
|
||||
|
||||
vllm_config.model_config.attn_type = "encoder_only"
|
||||
vllm_config.model_config.get_pooling_task.return_value = "embed&token_classify"
|
||||
assert PoolingRunner(model, vllm_config).supported_tasks == {
|
||||
"embed",
|
||||
"embed&token_classify",
|
||||
}
|
||||
@@ -1,280 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForMaskedLM, AutoModelForTokenClassification
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.models.utils import softmax
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def seed_everything():
|
||||
"""Seed all random number generators for reproducibility."""
|
||||
seed = 0
|
||||
set_random_seed(seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"boltuix/NeuroBERT-NER",
|
||||
],
|
||||
)
|
||||
# The float32 is required for this tiny model to pass the test.
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.core_model
|
||||
@torch.inference_mode
|
||||
def test_bert_model_runner_v2(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
prompt_batches = [[example_prompts[0]], example_prompts]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
vllm_output_batches = [
|
||||
vllm_model.token_classify(prompts) for prompts in prompt_batches
|
||||
]
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
hf_output_batches = [[hf_outputs[0]], hf_outputs]
|
||||
|
||||
# check logits difference
|
||||
for hf_outputs, vllm_outputs in zip(hf_output_batches, vllm_output_batches):
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_modernbert_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# NOTE: https://github.com/vllm-project/vllm/pull/32403
|
||||
# `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
|
||||
# model, which can cause numerical precision variance and edge cases.
|
||||
# We use @flaky(reruns=3) to mitigate intermittent failures.
|
||||
print(
|
||||
f"\n[NOTE] Testing {model} (randomly initialized weights) - "
|
||||
"flaky tolerance enabled due to numerical precision variance."
|
||||
)
|
||||
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_xlm_roberta_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
|
||||
|
||||
PRIVACY_FILTER_PROMPTS = [
|
||||
"My name is Harry Potter.",
|
||||
"Email me at harry.potter@hogwarts.edu.",
|
||||
"Call me on +44 20 7946 0958 tomorrow.",
|
||||
"My account number is 12345678 and the API key is sk-live-abc123def456.",
|
||||
"I live at 4 Privet Drive, Little Whinging.",
|
||||
"Visit https://example.com/profile/harry for more info.",
|
||||
"We met on 12 January 2024.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Workspace growth allocation issue on ROCm. See https://github.com/vllm-project/vllm/issues/48510",
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["openai/privacy-filter"])
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@torch.inference_mode
|
||||
def test_openai_privacy_filter(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForTokenClassification,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in PRIVACY_FILTER_PROMPTS:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@torch.inference_mode
|
||||
def test_auto_conversion(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForTokenClassification
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
|
||||
# check logits difference
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert torch.allclose(hf_output, vllm_output, atol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Original Google checkpoint: legacy `gamma`/`beta` LayerNorm names, an
|
||||
# NSP head (`cls.seq_relationship.*`) and a decoder tied to the input
|
||||
# embeddings (no explicit decoder weight in the checkpoint).
|
||||
"google-bert/bert-base-uncased",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_bert_for_masked_lm(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# BertForMaskedLM exposes its MLM head as a token-level pooling task; the
|
||||
# head applies softmax over the vocabulary, so each output row is a
|
||||
# distribution (matching HF's softmax(logits) below).
|
||||
with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_classify(example_prompts)
|
||||
|
||||
# Run hf_runner reference with "highest" fp32 precision to match
|
||||
# default behvior of vLLM. This is needed on ROCm since the
|
||||
# pooling tests set matmul precision to "high" in conftest.py
|
||||
prev_matmul_precision = torch.get_float32_matmul_precision()
|
||||
torch.set_float32_matmul_precision("highest")
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=AutoModelForMaskedLM,
|
||||
) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
inputs = tokenizer([prompt], return_tensors="pt")
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
hf_outputs.append(softmax(output.logits[0]))
|
||||
torch.set_float32_matmul_precision(prev_matmul_precision)
|
||||
|
||||
# Compare the per-token vocabulary distributions position by position.
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
hf_output = hf_output.detach().clone().cpu().float()
|
||||
vllm_output = vllm_output.detach().clone().cpu().float()
|
||||
assert hf_output.shape == vllm_output.shape
|
||||
assert torch.equal(hf_output.argmax(dim=-1), vllm_output.argmax(dim=-1))
|
||||
torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
|
||||
MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
|
||||
max_model_len = 128
|
||||
|
||||
input_str = """Immerse yourself in the enchanting chronicle of calculus, a
|
||||
mathematical domain that has radically transformed our comprehension of
|
||||
change and motion. Despite its roots in ancient civilizations, the
|
||||
formal birth of calculus predominantly occurred in the 17th century,
|
||||
primarily under the influential guidance of Sir Isaac Newton and Gottfried
|
||||
Wilhelm Leibniz. The earliest traces of calculus concepts are found in
|
||||
ancient Greek mathematics,most notably in the works of Eudoxus and
|
||||
Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
|
||||
technique for computing areas and volumes through the use of finite sums.
|
||||
This methodology laid crucial foundational work for integral calculus.
|
||||
In the 17th century, both Newton and Leibniz independently pioneered
|
||||
calculus, each contributing unique perspectives that would shape this new
|
||||
field."""
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def vllm_model(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME, runner="pooling", max_model_len=max_model_len
|
||||
) as model:
|
||||
yield model
|
||||
|
||||
|
||||
def test_smaller_truncation_size(
|
||||
vllm_model,
|
||||
):
|
||||
truncate_prompt_tokens = 10
|
||||
|
||||
vllm_output = vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
assert len(prompt_tokens) == truncate_prompt_tokens
|
||||
|
||||
|
||||
def test_max_truncation_size(vllm_model):
|
||||
truncate_prompt_tokens = -1
|
||||
|
||||
vllm_output = vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
|
||||
prompt_tokens = vllm_output[0].prompt_token_ids
|
||||
|
||||
# Pooling requests emit an embedding rather than generated tokens, so -1
|
||||
# truncates to the full context length.
|
||||
assert len(prompt_tokens) == max_model_len
|
||||
|
||||
|
||||
def test_bigger_truncation_size(vllm_model):
|
||||
truncate_prompt_tokens = max_model_len + 1
|
||||
|
||||
with pytest.raises(VLLMValidationError):
|
||||
vllm_model.llm.embed(
|
||||
input_str,
|
||||
tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
|
||||
)
|
||||
@@ -1,182 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Collective-safety tests for the Moondream3 MoE layer.
|
||||
|
||||
The MoE forward may fall back from the fused kernel to a Python expert loop.
|
||||
That decision is made per rank from a rank-local exception, so every path has
|
||||
to issue exactly one all-reduce. A path that issues two (or none) desyncs the
|
||||
tensor-parallel group and wedges it for the rest of the run.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import ensure_current_vllm_config
|
||||
from vllm.distributed.parallel_state import (
|
||||
destroy_distributed_environment,
|
||||
destroy_model_parallel,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.models import moondream3
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
HIDDEN_SIZE = 8
|
||||
EXPERT_INNER_DIM = 16
|
||||
NUM_EXPERTS = 4
|
||||
EXPERTS_PER_TOKEN = 2
|
||||
NUM_TOKENS = 3
|
||||
|
||||
|
||||
class _StubGate(torch.nn.Module):
|
||||
"""Router stand-in, so the test does not depend on platform-specific
|
||||
``ReplicatedLinear`` weight post-processing."""
|
||||
|
||||
def __init__(self, hidden_size: int, num_experts: int):
|
||||
super().__init__()
|
||||
self.linear = torch.nn.Linear(hidden_size, num_experts)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, None]:
|
||||
return self.linear(x), None
|
||||
|
||||
|
||||
class _CountingAllReduce:
|
||||
"""Stand-in for tensor_model_parallel_all_reduce that counts calls."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> torch.Tensor:
|
||||
self.calls += 1
|
||||
return x
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tp1_cpu():
|
||||
"""A single-rank CPU tensor-parallel group, enough to build the layer."""
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": "0",
|
||||
"LOCAL_RANK": "0",
|
||||
"WORLD_SIZE": "1",
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(get_open_port()),
|
||||
}
|
||||
)
|
||||
init_distributed_environment(backend="gloo")
|
||||
with ensure_current_vllm_config():
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
yield
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def moe(tp1_cpu) -> moondream3.Moondream3TextMoE:
|
||||
"""A single-rank MoE layer with deterministic weights."""
|
||||
layer = moondream3.Moondream3TextMoE(
|
||||
hidden_size=HIDDEN_SIZE,
|
||||
expert_inner_dim=EXPERT_INNER_DIM,
|
||||
num_experts=NUM_EXPERTS,
|
||||
experts_per_token=EXPERTS_PER_TOKEN,
|
||||
)
|
||||
layer.gate = _StubGate(HIDDEN_SIZE, NUM_EXPERTS)
|
||||
torch.nn.init.normal_(layer.fc1_weight, std=0.02)
|
||||
torch.nn.init.normal_(layer.fc2_weight, std=0.02)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def all_reduce(monkeypatch: pytest.MonkeyPatch) -> _CountingAllReduce:
|
||||
counter = _CountingAllReduce()
|
||||
monkeypatch.setattr(moondream3, "tensor_model_parallel_all_reduce", counter)
|
||||
return counter
|
||||
|
||||
|
||||
def test_expert_loop_all_reduces_once(moe, all_reduce):
|
||||
"""The Python expert loop issues exactly one all-reduce."""
|
||||
moe._use_fused_moe = False
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
|
||||
|
||||
def test_fused_path_all_reduces_once(moe, all_reduce, monkeypatch):
|
||||
"""The fused kernel path issues exactly one all-reduce."""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
monkeypatch.setattr(
|
||||
moondream3,
|
||||
"fused_experts",
|
||||
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
|
||||
)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
|
||||
|
||||
def test_all_reduce_failure_is_not_retried(moe, monkeypatch):
|
||||
"""A failing all-reduce must propagate, not trigger a second collective.
|
||||
|
||||
Regression test. The all-reduce used to sit inside the ``try`` guarding
|
||||
the fused kernel, so a ``RuntimeError`` raised by the collective itself
|
||||
was caught, and this rank went on to run the fallback loop and all-reduce
|
||||
a *second* time -- two collectives on one rank against one on each of its
|
||||
peers, which desyncs the tensor-parallel group and wedges it for good.
|
||||
"""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
monkeypatch.setattr(
|
||||
moondream3,
|
||||
"fused_experts",
|
||||
lambda **kwargs: torch.zeros_like(kwargs["hidden_states"]),
|
||||
)
|
||||
calls = []
|
||||
|
||||
def _failing_all_reduce(x: torch.Tensor) -> torch.Tensor:
|
||||
calls.append(x)
|
||||
raise RuntimeError("collective failed")
|
||||
|
||||
monkeypatch.setattr(
|
||||
moondream3, "tensor_model_parallel_all_reduce", _failing_all_reduce
|
||||
)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
with pytest.raises(RuntimeError, match="collective failed"):
|
||||
moe(x)
|
||||
|
||||
assert len(calls) == 1, (
|
||||
"the failed all-reduce was retried on the fallback path; peers issued "
|
||||
"one collective and this rank issued two"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exc", [RuntimeError("boom"), NotImplementedError()])
|
||||
def test_fallback_after_fused_failure_all_reduces_once(
|
||||
moe, all_reduce, monkeypatch, exc
|
||||
):
|
||||
"""A rank whose fused kernel fails still all-reduces exactly once."""
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
|
||||
def _raise(**kwargs):
|
||||
raise exc
|
||||
|
||||
monkeypatch.setattr(moondream3, "fused_experts", _raise)
|
||||
x = torch.randn(NUM_TOKENS, HIDDEN_SIZE)
|
||||
|
||||
out = moe(x)
|
||||
|
||||
assert out.shape == x.shape
|
||||
assert all_reduce.calls == 1
|
||||
# The fused path is disabled for subsequent forwards...
|
||||
assert moe._use_fused_moe is False
|
||||
# ...and those still all-reduce exactly once each.
|
||||
moe(x)
|
||||
assert all_reduce.calls == 2
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.model_executor.models.moss_audio import MOSS_AUDIO_PLACEHOLDER
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
CORE_MODEL = pytest.param(
|
||||
"OpenMOSS-Team/MOSS-Audio-4B-Instruct",
|
||||
marks=pytest.mark.core_model,
|
||||
id="4b-instruct",
|
||||
)
|
||||
|
||||
EXTENDED_MODELS = [
|
||||
"OpenMOSS-Team/MOSS-Audio-4B-Thinking",
|
||||
"OpenMOSS-Team/MOSS-Audio-8B-Instruct",
|
||||
"OpenMOSS-Team/MOSS-Audio-8B-Thinking",
|
||||
]
|
||||
|
||||
ACCURACY_MODELS = [CORE_MODEL, *EXTENDED_MODELS]
|
||||
|
||||
PARALLEL_SMOKE_CASES = [
|
||||
pytest.param({"tensor_parallel_size": 2}, id="tp2"),
|
||||
pytest.param({"pipeline_parallel_size": 2}, id="pp2"),
|
||||
pytest.param(
|
||||
{"tensor_parallel_size": 2, "pipeline_parallel_size": 2},
|
||||
id="tp2_pp2",
|
||||
),
|
||||
]
|
||||
|
||||
HF_ACCURACY_SKIP_REASON = (
|
||||
"HF AutoModelForCausalLM cannot load remote MOSS-Audio configs; "
|
||||
"vLLM generation coverage is provided by the smoke tests below."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
def test_moss_audio_generation_smoke(vllm_runner) -> None:
|
||||
model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
|
||||
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
max_model_len=1024,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
trust_remote_code=True,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(
|
||||
prompts,
|
||||
max_tokens=4,
|
||||
audios=audios,
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0][1]) > 0
|
||||
|
||||
|
||||
@pytest.mark.skip(reason=HF_ACCURACY_SKIP_REASON)
|
||||
@pytest.mark.parametrize("model", ACCURACY_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [8])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_moss_audio_hf_vllm_accuracy(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nTranscribe this audio."]
|
||||
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
max_model_len=1024,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
trust_remote_code=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=audios,
|
||||
)
|
||||
|
||||
with hf_runner(model, dtype=dtype, trust_remote_code=True) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=audios,
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("parallel_kwargs", PARALLEL_SMOKE_CASES)
|
||||
def test_moss_audio_parallel_smoke(vllm_runner, parallel_kwargs) -> None:
|
||||
model = "OpenMOSS-Team/MOSS-Audio-4B-Instruct"
|
||||
required_gpus = parallel_kwargs.get(
|
||||
"tensor_parallel_size", 1
|
||||
) * parallel_kwargs.get("pipeline_parallel_size", 1)
|
||||
if current_platform.device_count() < required_gpus:
|
||||
# TP/PP integration smoke runs on local or multi-GPU CI only.
|
||||
pytest.skip(f"Requires at least {required_gpus} GPUs")
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
prompts = [f"{MOSS_AUDIO_PLACEHOLDER}\nBriefly describe this audio."]
|
||||
audios = [[AudioAsset("mary_had_lamb").audio_and_sample_rate[0]]]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
max_model_len=1024,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
trust_remote_code=True,
|
||||
**parallel_kwargs,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(
|
||||
prompts,
|
||||
max_tokens=4,
|
||||
audios=audios,
|
||||
)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert len(outputs[0][1]) > 0
|
||||
@@ -1,225 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.model_executor.models.pixtral import _make_packed_sequence_metadata
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from ....utils import VLLM_PATH, large_gpu_test
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import StrPath
|
||||
|
||||
PIXTRAL_ID = "mistralai/Pixtral-12B-2409"
|
||||
MISTRAL_SMALL_3_1_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
|
||||
MINISTRAL_3B_ID = "mistralai/Ministral-3-3B-Instruct-2512"
|
||||
|
||||
MODELS = [PIXTRAL_ID, MISTRAL_SMALL_3_1_ID]
|
||||
|
||||
IMG_URLS = [
|
||||
"237-400x300.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
|
||||
"231-200x300.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
|
||||
"27-500x500.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
|
||||
"17-150x600.jpg", # "https://huggingface.co/datasets/Isotr0py/mistral-test-images/resolve/main/237-400x300.jpg",
|
||||
]
|
||||
PROMPT = "Describe each image in one short sentence."
|
||||
|
||||
|
||||
def _create_msg_format(urls: list[str]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": PROMPT,
|
||||
}
|
||||
]
|
||||
+ [{"type": "image_url", "image_url": {"url": url}} for url in urls],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
|
||||
LIMIT_MM_PER_PROMPT = dict(image=4)
|
||||
|
||||
MAX_MODEL_LEN = [8192, 65536]
|
||||
|
||||
FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
|
||||
assert FIXTURES_PATH.exists()
|
||||
|
||||
FIXTURE_LOGPROBS_CHAT = {
|
||||
PIXTRAL_ID: FIXTURES_PATH / "pixtral_chat.json",
|
||||
MISTRAL_SMALL_3_1_ID: FIXTURES_PATH / "mistral_small_3_chat.json",
|
||||
MINISTRAL_3B_ID: FIXTURES_PATH / "ministral_3b_chat.json",
|
||||
}
|
||||
|
||||
OutputsLogprobs = list[tuple[list[int], str, SampleLogprobs | None]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
],
|
||||
)
|
||||
def test_packed_sequence_metadata(backend: AttentionBackendEnum) -> None:
|
||||
cu_seqlens, max_seqlen, sequence_lengths = _make_packed_sequence_metadata(
|
||||
[4, 6],
|
||||
backend,
|
||||
hidden_size=64,
|
||||
tp_size=1,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
|
||||
assert cu_seqlens.dtype == torch.int32
|
||||
assert max_seqlen.dtype == torch.int32
|
||||
if backend == AttentionBackendEnum.FLASHINFER:
|
||||
assert max_seqlen.item() >= 6
|
||||
assert sequence_lengths is not None
|
||||
assert sequence_lengths.dtype == torch.int32
|
||||
assert len(cu_seqlens) % 2 == 0
|
||||
else:
|
||||
expected_max_seqlen = 6 if backend == AttentionBackendEnum.FLASH_ATTN else 0
|
||||
assert max_seqlen.item() == expected_max_seqlen
|
||||
assert cu_seqlens.tolist() == [0, 4, 10]
|
||||
assert sequence_lengths is None
|
||||
|
||||
|
||||
# For the test author to store golden output in JSON
|
||||
def _dump_outputs_w_logprobs(
|
||||
outputs: OutputsLogprobs,
|
||||
filename: "StrPath",
|
||||
) -> None:
|
||||
json_data = [
|
||||
(
|
||||
tokens,
|
||||
text,
|
||||
[
|
||||
{k: asdict(v) for k, v in token_logprobs.items()}
|
||||
for token_logprobs in (logprobs or [])
|
||||
],
|
||||
)
|
||||
for tokens, text, logprobs in outputs
|
||||
]
|
||||
|
||||
with open(filename, "w") as f:
|
||||
json.dump(json_data, f)
|
||||
|
||||
|
||||
def load_outputs_w_logprobs(filename: "StrPath") -> OutputsLogprobs:
|
||||
with open(filename, "rb") as f:
|
||||
json_data = json.load(f)
|
||||
|
||||
return [
|
||||
(
|
||||
tokens,
|
||||
text,
|
||||
[
|
||||
{int(k): Logprob(**v) for k, v in token_logprobs.items()}
|
||||
for token_logprobs in logprobs
|
||||
],
|
||||
)
|
||||
for tokens, text, logprobs in json_data
|
||||
]
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=80)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("max_model_len", MAX_MODEL_LEN)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_chat(
|
||||
vllm_runner, max_model_len: int, model: str, dtype: str, local_asset_server
|
||||
) -> None:
|
||||
if (
|
||||
model == MISTRAL_SMALL_3_1_ID
|
||||
and max_model_len == 65536
|
||||
and current_platform.is_rocm()
|
||||
):
|
||||
pytest.skip(
|
||||
"OOM on ROCm: 24B model with 65536 context length exceeds GPU memory"
|
||||
)
|
||||
|
||||
EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(FIXTURE_LOGPROBS_CHAT[model])
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="mistral",
|
||||
load_format="mistral",
|
||||
config_format="mistral",
|
||||
max_model_len=max_model_len,
|
||||
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
|
||||
) as vllm_model:
|
||||
outputs = []
|
||||
|
||||
urls_all = [local_asset_server.url_for(u) for u in IMG_URLS]
|
||||
msgs = [
|
||||
_create_msg_format(urls_all[:1]),
|
||||
_create_msg_format(urls_all[:2]),
|
||||
_create_msg_format(urls_all),
|
||||
]
|
||||
for msg in msgs:
|
||||
output = vllm_model.llm.chat(msg, sampling_params=SAMPLING_PARAMS)
|
||||
|
||||
outputs.extend(output)
|
||||
|
||||
logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
|
||||
# Remove last `None` prompt_logprobs to compare with fixture
|
||||
for i in range(len(logprobs)):
|
||||
assert logprobs[i][-1] is None
|
||||
logprobs[i] = logprobs[i][:-1]
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=EXPECTED_CHAT_LOGPROBS,
|
||||
outputs_1_lst=logprobs,
|
||||
name_0="h100_ref",
|
||||
name_1="output",
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=16)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_chat_consolidated(vllm_runner, dtype: str, local_asset_server) -> None:
|
||||
EXPECTED_CHAT_LOGPROBS = load_outputs_w_logprobs(
|
||||
FIXTURE_LOGPROBS_CHAT[MINISTRAL_3B_ID]
|
||||
)
|
||||
with vllm_runner(
|
||||
MINISTRAL_3B_ID,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="mistral",
|
||||
load_format="mistral",
|
||||
config_format="mistral",
|
||||
max_model_len=8192,
|
||||
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
|
||||
) as vllm_model:
|
||||
outputs = []
|
||||
urls_all = [local_asset_server.url_for(u) for u in IMG_URLS]
|
||||
msgs = [
|
||||
_create_msg_format(urls_all[:1]),
|
||||
_create_msg_format(urls_all[:2]),
|
||||
_create_msg_format(urls_all),
|
||||
]
|
||||
for msg in msgs:
|
||||
output = vllm_model.llm.chat(msg, sampling_params=SAMPLING_PARAMS)
|
||||
outputs.extend(output)
|
||||
|
||||
logprobs = vllm_runner._final_steps_generate_w_logprobs(outputs)
|
||||
for i in range(len(logprobs)):
|
||||
assert logprobs[i][-1] is None
|
||||
logprobs[i] = logprobs[i][:-1]
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=EXPECTED_CHAT_LOGPROBS,
|
||||
outputs_1_lst=logprobs,
|
||||
name_0="h100_ref",
|
||||
name_1="output",
|
||||
)
|
||||
@@ -1,357 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from fractions import Fraction
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from mistral_common.protocol.transcription.request import (
|
||||
StreamingMode,
|
||||
TranscriptionRequest,
|
||||
)
|
||||
from mistral_common.tokens.tokenizers.audio import Audio
|
||||
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
|
||||
from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import StreamingInput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.kv_cache_interface import SlidingWindowSpec
|
||||
|
||||
from ....utils import ROCM_ENGINE_KWARGS, create_new_process_for_each_test
|
||||
|
||||
MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
|
||||
AUDIO_LAYER_NAME = "whisper_encoder.whisper_encoder.layers.0.layers.self_attn.attn"
|
||||
ENGINE_CONFIG = {
|
||||
"model": MODEL_NAME,
|
||||
"max_model_len": 8192,
|
||||
"max_num_seqs": 4,
|
||||
"limit_mm_per_prompt": {"audio": 1},
|
||||
"config_format": "mistral",
|
||||
"load_format": "mistral",
|
||||
"tokenizer_mode": "mistral",
|
||||
"enforce_eager": True,
|
||||
"gpu_memory_utilization": 0.9,
|
||||
**ROCM_ENGINE_KWARGS,
|
||||
}
|
||||
|
||||
|
||||
EXPECTED_TEXT = [
|
||||
(
|
||||
" First words I spoke in the original phonograph. "
|
||||
"A little piece of practical poetry. Mary had a little lamb,"
|
||||
" its fleece was quite a slow, and everywhere that Mary went, "
|
||||
"the lamb was sure to go."
|
||||
),
|
||||
(
|
||||
" And the 0-1 pitch on the way to Edgar Martinez. Swung on"
|
||||
" the line. Down the left field line for OBS. Here comes Joy. "
|
||||
"Here is Junior to third base. They're going to wave him in. "
|
||||
"The throw to the plate will be late. The Mariners are going"
|
||||
" to play. For the American League Championship, "
|
||||
"I don't believe it. It just continues. My, oh, my."
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _normalize(texts: list[str]) -> list[str]:
|
||||
# The model occasionally transcribes "OBS" as "a base hit" and
|
||||
# "oh, my" as "oh my", but both are acoustically valid. Normalise so
|
||||
# the assertion is stable across runs and hardware.
|
||||
texts[1] = texts[1].replace("a base hit", "OBS").replace("oh my", "oh, my")
|
||||
return texts
|
||||
|
||||
|
||||
def assert_encoder_kv_cache_spec(engine: LLM) -> None:
|
||||
vllm_config = engine.llm_engine.vllm_config
|
||||
audio_config = vllm_config.model_config.hf_config.audio_config
|
||||
kv_cache_specs_per_rank = engine.llm_engine.model_executor.get_kv_cache_specs()
|
||||
|
||||
assert len(kv_cache_specs_per_rank) == 1
|
||||
kv_cache_specs = kv_cache_specs_per_rank[0]
|
||||
assert AUDIO_LAYER_NAME in kv_cache_specs, kv_cache_specs.keys()
|
||||
spec = kv_cache_specs[AUDIO_LAYER_NAME]
|
||||
|
||||
assert audio_config.sliding_window == 750
|
||||
assert audio_config.block_pool_size == 4
|
||||
assert isinstance(spec, SlidingWindowSpec)
|
||||
assert spec.block_size == 16
|
||||
assert spec.num_kv_heads == 32
|
||||
assert spec.tokens_per_state == Fraction(1, 4)
|
||||
# cdiv(750, 4) == 188 pooled tokens cover the model's window; the extra
|
||||
# +1 is an eviction margin (see whisper_causal.py get_kv_cache_spec).
|
||||
assert spec.sliding_window == cdiv(750, 4) + 1 == 189
|
||||
assert (
|
||||
spec.max_admission_blocks_per_request(
|
||||
max_in_flight_tokens=1,
|
||||
max_model_len=vllm_config.model_config.max_model_len,
|
||||
)
|
||||
== 13
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def audio_assets() -> list[AudioAsset]:
|
||||
return [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer() -> MistralTokenizer:
|
||||
return MistralTokenizer.from_hf_hub(MODEL_NAME)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def async_engine(request):
|
||||
engine_config = {**ENGINE_CONFIG, **getattr(request, "param", {})}
|
||||
gpu_memory_utilization = ENGINE_CONFIG.get("gpu_memory_utilization", 0.9)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
engine_args = AsyncEngineArgs(**engine_config)
|
||||
llm = AsyncLLM.from_engine_args(engine_args)
|
||||
try:
|
||||
yield llm
|
||||
finally:
|
||||
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
|
||||
llm.shutdown(timeout=shutdown_timeout)
|
||||
del llm
|
||||
import torch
|
||||
|
||||
torch._dynamo.reset()
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
|
||||
wait_for_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
|
||||
|
||||
|
||||
def _build_inputs(
|
||||
audio_assets: list[AudioAsset], tokenizer: MistralTokenizer
|
||||
) -> tuple[list[dict], list[SamplingParams]]:
|
||||
audio_config = tokenizer.instruct_tokenizer.tokenizer.audio
|
||||
inputs = []
|
||||
sampling_params = []
|
||||
for audio_asset in audio_assets:
|
||||
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
audio=audio.to_base64(audio.format),
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
language=None,
|
||||
)
|
||||
tokenized = tokenizer.instruct_tokenizer.encode_transcription(req)
|
||||
tokens = tokenized.tokens
|
||||
audio_array = tokenized.audios[0].audio_array
|
||||
|
||||
num_samples = audio_array.shape[0]
|
||||
max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1
|
||||
sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens))
|
||||
inputs.append(
|
||||
{
|
||||
"multi_modal_data": {"audio": [(audio_array, None)]},
|
||||
"prompt_token_ids": tokens,
|
||||
}
|
||||
)
|
||||
return inputs, sampling_params
|
||||
|
||||
|
||||
def _assert_expected_text(outputs) -> None:
|
||||
texts = _normalize([out.outputs[0].text for out in outputs])
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
vllm_kwargs = {**ENGINE_CONFIG}
|
||||
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
|
||||
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
assert_encoder_kv_cache_spec(vllm_model.llm)
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="Full cudagraphs require CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"cudagraph_mode",
|
||||
[CUDAGraphMode.FULL_DECODE_ONLY, CUDAGraphMode.FULL_AND_PIECEWISE],
|
||||
)
|
||||
@create_new_process_for_each_test(method="spawn")
|
||||
def test_voxtral_realtime_cudagraph(
|
||||
audio_assets, tokenizer, vllm_runner, monkeypatch, cudagraph_mode
|
||||
):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
vllm_kwargs = {**ENGINE_CONFIG}
|
||||
vllm_kwargs["model_name"] = vllm_kwargs.pop("model")
|
||||
vllm_kwargs["enforce_eager"] = False
|
||||
vllm_kwargs["compilation_config"] = {
|
||||
"cudagraph_mode": cudagraph_mode,
|
||||
# max_num_seqs is 4, so this is the only decode size worth capturing.
|
||||
"cudagraph_capture_sizes": [4],
|
||||
}
|
||||
|
||||
with vllm_runner(**vllm_kwargs) as vllm_model:
|
||||
inputs, sampling_params = _build_inputs(audio_assets, tokenizer)
|
||||
|
||||
outputs = vllm_model.llm.generate(inputs, sampling_params=sampling_params)
|
||||
_assert_expected_text(outputs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine):
|
||||
# Lazy import to avoid CUDA-reinitialization error
|
||||
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
|
||||
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||
|
||||
async def input_stream(audio_buffer):
|
||||
async for prompt in audio_buffer.get_input_stream():
|
||||
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
|
||||
(engine_input,) = await async_engine.renderer.render_cmpl_async(
|
||||
[parsed_prompt]
|
||||
)
|
||||
yield StreamingInput(prompt=engine_input)
|
||||
|
||||
output_tokens_list = []
|
||||
for i, audio_asset in enumerate(audio_assets):
|
||||
output_tokens = []
|
||||
audio = Audio.from_file(audio_asset.get_local_path(), strict=False)
|
||||
|
||||
req = TranscriptionRequest(
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
audio=audio.to_base64(audio.format),
|
||||
language=None,
|
||||
)
|
||||
audio_enc = tokenizer.encode_transcription(req)
|
||||
|
||||
buffer = VoxtralRealtimeBuffer(audio_config, audio_enc.tokens)
|
||||
await buffer.append_audio(audio_enc.audios[0].audio_array)
|
||||
await buffer.append_audio(None)
|
||||
|
||||
request_id = f"session-{i}"
|
||||
|
||||
async for resp in async_engine.generate(
|
||||
prompt=input_stream(buffer),
|
||||
sampling_params=sampling_params,
|
||||
request_id=request_id,
|
||||
):
|
||||
tokens = resp.outputs[0].token_ids[-1:]
|
||||
output_tokens.extend(tokens)
|
||||
await buffer.append_tokens(tokens)
|
||||
|
||||
output_tokens_list.append(output_tokens)
|
||||
|
||||
texts = _normalize(
|
||||
[
|
||||
tokenizer.decode(
|
||||
output_tokens, special_token_policy=SpecialTokenPolicy.IGNORE
|
||||
)
|
||||
for output_tokens in output_tokens_list
|
||||
]
|
||||
)
|
||||
for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)):
|
||||
assert got == expected, (
|
||||
f"Output mismatch at index {i}:\n"
|
||||
f" got: {got!r}\n"
|
||||
f" expected: {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_voxtral_realtime_buffer_stops_at_max_model_len(tokenizer):
|
||||
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
|
||||
|
||||
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||
audio = Audio.from_file(AudioAsset("winning_call").get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
audio=audio.to_base64(audio.format),
|
||||
language=None,
|
||||
)
|
||||
audio_enc = tokenizer.encode_transcription(req)
|
||||
|
||||
max_model_len = 64
|
||||
buffer = VoxtralRealtimeBuffer(
|
||||
audio_config, audio_enc.tokens, max_model_len=max_model_len
|
||||
)
|
||||
await buffer.append_audio(audio_enc.audios[0].audio_array)
|
||||
await buffer.append_audio(None)
|
||||
# Unlimited feedback tokens so only the length cap can stop the stream.
|
||||
await buffer.append_tokens([0] * max_model_len)
|
||||
|
||||
n_tokens = 0
|
||||
async for prompt in buffer.get_input_stream():
|
||||
n_tokens += len(prompt["prompt_token_ids"])
|
||||
|
||||
assert len(audio_enc.tokens) < n_tokens < max_model_len
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("async_engine", [{"max_model_len": 128}], indirect=True)
|
||||
async def test_voxtral_realtime_generator_max_model_len(
|
||||
audio_assets, tokenizer, async_engine
|
||||
):
|
||||
# Lazy import to avoid CUDA-reinitialization error
|
||||
from vllm.model_executor.models.voxtral_realtime import VoxtralRealtimeBuffer
|
||||
|
||||
max_model_len = async_engine.model_config.max_model_len
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=1)
|
||||
audio_config = tokenizer.instruct_tokenizer.audio_encoder.audio_config
|
||||
|
||||
async def input_stream(audio_buffer):
|
||||
async for prompt in audio_buffer.get_input_stream():
|
||||
parsed_prompt = parse_model_prompt(async_engine.model_config, prompt)
|
||||
(engine_input,) = await async_engine.renderer.render_cmpl_async(
|
||||
[parsed_prompt]
|
||||
)
|
||||
yield StreamingInput(prompt=engine_input)
|
||||
|
||||
# winning_call is ~300 tokens long, well past max_model_len.
|
||||
audio = Audio.from_file(audio_assets[1].get_local_path(), strict=False)
|
||||
req = TranscriptionRequest(
|
||||
streaming=StreamingMode.OFFLINE,
|
||||
audio=audio.to_base64(audio.format),
|
||||
language=None,
|
||||
)
|
||||
audio_enc = tokenizer.encode_transcription(req)
|
||||
buffer = VoxtralRealtimeBuffer(
|
||||
audio_config, audio_enc.tokens, max_model_len=max_model_len
|
||||
)
|
||||
await buffer.append_audio(audio_enc.audios[0].audio_array)
|
||||
await buffer.append_audio(None)
|
||||
|
||||
output_tokens = []
|
||||
async for resp in async_engine.generate(
|
||||
prompt=input_stream(buffer),
|
||||
sampling_params=sampling_params,
|
||||
request_id="session-max-len",
|
||||
):
|
||||
tokens = resp.outputs[0].token_ids[-1:]
|
||||
output_tokens.extend(tokens)
|
||||
await buffer.append_tokens(tokens)
|
||||
|
||||
assert resp.finished
|
||||
assert 0 < len(output_tokens) < max_model_len
|
||||
# The engine is still healthy after the session hit the cap.
|
||||
assert not async_engine.errored
|
||||
@@ -1,418 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForSpeechSeq2Seq
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import HfRunner, PromptAudioInput, VllmRunner
|
||||
from ....utils import create_new_process_for_each_test, multi_gpu_test
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
VLLM_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
|
||||
HF_PROMPT = ""
|
||||
# Whisper expects 16kHz audio
|
||||
WHISPER_SAMPLE_RATE = 16000
|
||||
BEAM_WIDTHS = (1, 2)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_spawn_for_whisper(monkeypatch):
|
||||
"""Whisper has issues with forked workers, use spawn instead."""
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
def run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: Sequence[tuple[list[str], list[str], PromptAudioInput]],
|
||||
model: str,
|
||||
*,
|
||||
max_model_len: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
enforce_eager: bool = True,
|
||||
gpu_memory_utilization: float = 0.9,
|
||||
) -> None:
|
||||
"""Inference result should be the same between hf and vllm.
|
||||
|
||||
All the audio fixtures for the test are from AudioAsset.
|
||||
For huggingface runner, we provide the audio as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding MultiModalConfig as input.
|
||||
"""
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
enforce_eager=enforce_eager,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
disable_custom_all_reduce=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
vllm_prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=audios,
|
||||
)
|
||||
for vllm_prompts, _, audios in inputs
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
hf_prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=audios,
|
||||
)
|
||||
for _, hf_prompts, audios in inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def resampled_assets() -> list[tuple[Any, int]]:
|
||||
audio_assets = [AudioAsset("mary_had_lamb"), AudioAsset("winning_call")]
|
||||
sampled_assets = []
|
||||
resampler = AudioResampler(target_sr=WHISPER_SAMPLE_RATE)
|
||||
for asset in audio_assets:
|
||||
audio, orig_sr = asset.audio_and_sample_rate
|
||||
# Resample to Whisper's expected sample rate (16kHz)
|
||||
if orig_sr != WHISPER_SAMPLE_RATE:
|
||||
audio = resampler.resample(audio, orig_sr=orig_sr)
|
||||
sampled_assets.append(
|
||||
(audio, WHISPER_SAMPLE_RATE),
|
||||
)
|
||||
return sampled_assets
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def input_audios(
|
||||
resampled_assets,
|
||||
) -> list[tuple[list[str], list[str], list[tuple[Any, int]]]]:
|
||||
inputs = []
|
||||
# audio assets are resampled to WHISPER_SAMPLE_RATE
|
||||
for audio_info in resampled_assets:
|
||||
# vLLM prompts, HF prompts, audio inputs
|
||||
inputs.append(([VLLM_PROMPT], [HF_PROMPT], [audio_info]))
|
||||
return inputs
|
||||
|
||||
|
||||
def check_model_available(model: str) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
def test_beam_search_encoder_decoder(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
resampled_assets,
|
||||
) -> None:
|
||||
"""Test beam search with encoder-decoder models (Whisper)."""
|
||||
model = "openai/whisper-large-v3-turbo"
|
||||
check_model_available(model)
|
||||
|
||||
hf_prompts = [
|
||||
"<|startoftranscript|>",
|
||||
"<|startoftranscript|>",
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
|
||||
hf_outputs_by_beam_width = [
|
||||
hf_model.generate_beam_search(
|
||||
hf_prompts,
|
||||
beam_width=beam_width,
|
||||
max_tokens=max_tokens,
|
||||
audios=resampled_assets,
|
||||
)
|
||||
for beam_width in BEAM_WIDTHS
|
||||
]
|
||||
|
||||
# Test both explicit encoder/decoder prompts
|
||||
vllm_prompts = [
|
||||
# Implicit encoder/decoder prompt
|
||||
{
|
||||
"prompt": "<|startoftranscript|>",
|
||||
"multi_modal_data": {"audio": resampled_assets[0]},
|
||||
},
|
||||
# Explicit encoder/decover prompt
|
||||
{
|
||||
"encoder_prompt": {
|
||||
"prompt": "",
|
||||
"multi_modal_data": {"audio": resampled_assets[1]},
|
||||
},
|
||||
"decoder_prompt": "<|startoftranscript|>",
|
||||
},
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype="half",
|
||||
max_model_len=448,
|
||||
tensor_parallel_size=1,
|
||||
max_num_seqs=4,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_by_beam_width = [
|
||||
vllm_model.generate_beam_search(
|
||||
vllm_prompts,
|
||||
beam_width=beam_width,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
for beam_width in BEAM_WIDTHS
|
||||
]
|
||||
|
||||
for beam_width, hf_outputs, vllm_outputs in zip(
|
||||
BEAM_WIDTHS, hf_outputs_by_beam_width, vllm_outputs_by_beam_width
|
||||
):
|
||||
for i in range(len(vllm_prompts)):
|
||||
hf_output_ids, hf_output_texts = hf_outputs[i]
|
||||
vllm_output_ids, vllm_output_texts = vllm_outputs[i]
|
||||
|
||||
for j, (hf_text, vllm_text) in enumerate(
|
||||
zip(hf_output_texts, vllm_output_texts)
|
||||
):
|
||||
print(f">>>{j}-th hf output [NOTE: special tokens are filtered]:")
|
||||
print(hf_text)
|
||||
print(f">>>{j}-th vllm output:")
|
||||
print(vllm_text)
|
||||
|
||||
# Check that we got the same number of beams
|
||||
assert len(hf_output_ids) == len(vllm_output_ids) == beam_width
|
||||
|
||||
# For encoder-decoder models, we primarily want to verify that:
|
||||
# 1. Beam search completes without errors
|
||||
# 2. We get the expected number of beams
|
||||
# 3. Outputs are reasonable (non-empty, diverse beams)
|
||||
for j in range(len(vllm_output_ids)):
|
||||
# Check that outputs are not empty
|
||||
assert len(vllm_output_ids[j]) > 0, (
|
||||
f"Prompt {i}, beam {j}: empty output"
|
||||
)
|
||||
# Check that decoded text is not empty
|
||||
assert len(vllm_output_texts[j].strip()) > 0, (
|
||||
f"Prompt {i}, beam {j}: empty text output"
|
||||
)
|
||||
|
||||
|
||||
def test_parse_language_detection_output():
|
||||
"""Unit test for WhisperForConditionalGeneration.parse_language_detection_output.
|
||||
|
||||
No GPU or model loading required.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.model_executor.models.whisper import (
|
||||
WhisperForConditionalGeneration,
|
||||
)
|
||||
|
||||
cls = WhisperForConditionalGeneration
|
||||
|
||||
def make_tokenizer(return_value: str) -> MagicMock:
|
||||
tok = MagicMock()
|
||||
tok.decode = MagicMock(return_value=return_value)
|
||||
return tok
|
||||
|
||||
# English
|
||||
assert (
|
||||
cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en"
|
||||
)
|
||||
|
||||
# German
|
||||
assert (
|
||||
cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de"
|
||||
)
|
||||
|
||||
# Unsupported language code
|
||||
with pytest.raises(AssertionError):
|
||||
cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>"))
|
||||
|
||||
# No special token format
|
||||
with pytest.raises(AssertionError):
|
||||
cls.parse_language_detection_output([1], make_tokenizer("hello"))
|
||||
|
||||
# Empty token_ids
|
||||
with pytest.raises((AssertionError, IndexError)):
|
||||
cls.parse_language_detection_output([], make_tokenizer("anything"))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_injects_out_features():
|
||||
"""Packed k_proj weights get a zeros bias of length ``out_features``."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
packed = torch.ones(2, 16)
|
||||
result = dict(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)],
|
||||
".k_proj.weight_packed",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
bias_name = "model.encoder.layers.0.self_attn.k_proj.bias"
|
||||
assert bias_name in result
|
||||
assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result
|
||||
assert tuple(result[bias_name].shape) == (8,)
|
||||
assert torch.equal(result[bias_name], torch.zeros(8))
|
||||
|
||||
|
||||
def test_create_fake_bias_for_k_proj_forwards_real_bias():
|
||||
"""A checkpoint bias is forwarded and no extra fake bias is emitted."""
|
||||
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
|
||||
|
||||
weight = torch.ones(8, 16)
|
||||
real_bias = torch.ones(8)
|
||||
items = list(
|
||||
_create_fake_bias_for_k_proj(
|
||||
[
|
||||
("layers.0.self_attn.k_proj.weight", weight),
|
||||
("layers.0.self_attn.k_proj.bias", real_bias),
|
||||
],
|
||||
".k_proj.weight",
|
||||
out_features=8,
|
||||
)
|
||||
)
|
||||
names = [n for n, _ in items]
|
||||
assert names == [
|
||||
"layers.0.self_attn.k_proj.weight",
|
||||
"layers.0.self_attn.k_proj.bias",
|
||||
]
|
||||
assert items[1][1] is real_bias
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
@pytest.mark.parametrize("dtype", ["half", "float"])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
num_logprobs: int,
|
||||
input_audios,
|
||||
enforce_eager: bool,
|
||||
) -> None:
|
||||
check_model_available(model)
|
||||
if current_platform.is_cpu() and not enforce_eager:
|
||||
pytest.skip("Skipping test for CPU with non-eager mode")
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_audios,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=448,
|
||||
max_tokens=200,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=1,
|
||||
enforce_eager=enforce_eager,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [200])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_models_distributed(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
distributed_executor_backend: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
input_audios,
|
||||
) -> None:
|
||||
check_model_available(model)
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_audios,
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=448,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=2,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=False,
|
||||
gpu_memory_utilization=0.65,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
|
||||
def test_encoder_cache_cleanup(
|
||||
vllm_runner,
|
||||
model: str,
|
||||
input_audios,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Test that encoder cache is properly cleaned up after requests complete.
|
||||
|
||||
This is a regression test for a bug where encoder cache entries were freed
|
||||
in the same scheduling step they were allocated, before the model could use
|
||||
them.
|
||||
"""
|
||||
# Set single-process mode to access the model runner's encoder cache directly
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
check_model_available(model)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype="half",
|
||||
max_model_len=448,
|
||||
tensor_parallel_size=1,
|
||||
limit_mm_per_prompt={"audio": 2},
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
engine_core = vllm_model.llm.llm_engine.engine_core.engine_core
|
||||
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
|
||||
encoder_cache = model_runner.encoder_cache
|
||||
|
||||
# Run multiple sequential requests to ensure cache is properly managed
|
||||
for vllm_prompts, _, audios in input_audios:
|
||||
vllm_model.generate_greedy(vllm_prompts, max_tokens=50, audios=audios)
|
||||
|
||||
# After all requests complete, encoder cache should be empty
|
||||
cache_size = len(encoder_cache)
|
||||
assert cache_size == 0, (
|
||||
f"Encoder cache should be empty after all requests complete, "
|
||||
f"but has {cache_size} entries. This indicates encoder cache "
|
||||
f"entries are not being properly freed."
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Pytest configuration for vLLM multimodal pooling tests."""
|
||||
|
||||
import gc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import wait_for_memory_to_settle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def release_gpu_memory_between_tests():
|
||||
"""Reclaim cached GPU memory between tests in this shard.
|
||||
|
||||
These tests start many vLLM/HF engines sequentially in one pytest
|
||||
process; without explicit reclamation between tests, VRAM fragmentation
|
||||
accumulates and later tests fail at engine startup ("Engine core
|
||||
initialization failed").
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm():
|
||||
return
|
||||
try:
|
||||
wait_for_memory_to_settle()
|
||||
except ValueError as e:
|
||||
# Longer-lived (class/module-scoped) engine fixtures may legitimately
|
||||
# still hold VRAM; nothing further to reclaim in that case.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
@@ -1,174 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
|
||||
which loops over questions where the pooler batches them."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.clef_joint_schema import (
|
||||
JointSchemaHead,
|
||||
JointSchemaPooler,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
HIDDEN, VOCAB = 64, 500
|
||||
TYPES = [0, 1, 1, 2]
|
||||
COUNTS = [2, 3, 1, 5]
|
||||
|
||||
|
||||
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
|
||||
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
|
||||
for one record, with ``spans`` the option spans of each question."""
|
||||
hidden = head.hidden_norm(hidden)
|
||||
memory = head.memory_projection(hidden).unsqueeze(0)
|
||||
global_vector = hidden[-1]
|
||||
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
|
||||
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
|
||||
lexicals = [
|
||||
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
|
||||
]
|
||||
options = torch.cat(
|
||||
[
|
||||
head.option_context_projection(c)
|
||||
+ head.option_lexical_projection(lex)
|
||||
+ head.option_question_projection(questions[i]).unsqueeze(0)
|
||||
for i, (c, lex) in enumerate(zip(contexts, lexicals))
|
||||
]
|
||||
).unsqueeze(0)
|
||||
for layer in head.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
routed = torch.split(options[0], [len(q) for q in spans])
|
||||
|
||||
base = head.question_projection(questions)
|
||||
summaries = []
|
||||
for field, opts in zip(base, routed):
|
||||
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
|
||||
summaries.append((weights[:, None] * opts).sum(0))
|
||||
fields = (
|
||||
base
|
||||
+ head.option_summary_norm(torch.stack(summaries))
|
||||
+ head.global_projection(global_vector)
|
||||
+ head.type_embedding(torch.tensor(TYPES, device=hidden.device))
|
||||
).unsqueeze(0)
|
||||
for layer in head.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = head.field_norm(fields[0])
|
||||
|
||||
logits = []
|
||||
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
|
||||
anchor = F.normalize(questions[i] + global_vector, dim=-1)
|
||||
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
|
||||
opts = head.option_norm(opts)
|
||||
field = field.expand_as(opts)
|
||||
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
|
||||
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
|
||||
joint = joint + head.residual_scorer(features).squeeze(-1)
|
||||
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pooler():
|
||||
torch.manual_seed(0)
|
||||
head = JointSchemaHead(
|
||||
hidden_size=HIDDEN,
|
||||
width=32,
|
||||
routing_layers=2,
|
||||
layers=2,
|
||||
heads=4,
|
||||
feedforward=48,
|
||||
).eval()
|
||||
with torch.no_grad():
|
||||
for p in head.parameters():
|
||||
p.add_(torch.randn_like(p) * 0.2)
|
||||
head.to(DEVICE_TYPE)
|
||||
embedding = torch.randn(VOCAB, HIDDEN, device=DEVICE_TYPE)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
enable_chunked_prefill=False, async_scheduling=False
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=config,
|
||||
):
|
||||
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
|
||||
return pooler, embedding
|
||||
|
||||
|
||||
def schema(length: int):
|
||||
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
|
||||
pos = 20
|
||||
question_spans: list[tuple[int, int]] = []
|
||||
spans: list[list[tuple[int, int]]] = []
|
||||
for count in COUNTS:
|
||||
question_spans.append((pos, pos + 4))
|
||||
pos += 6
|
||||
spans.append([])
|
||||
for k in range(count):
|
||||
n = 1 + (k * 3) % 4
|
||||
spans[-1].append((pos, pos + n))
|
||||
pos += n + 1
|
||||
assert pos < length
|
||||
extra = {
|
||||
"question_types": TYPES,
|
||||
"question_spans": [[s - length, e - length] for s, e in question_spans],
|
||||
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
|
||||
"option_counts": COUNTS,
|
||||
}
|
||||
return {"joint_schema": extra}, question_spans, spans
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_pooler_matches_reference(pooler):
|
||||
pooler, embedding = pooler
|
||||
lengths = [120, 90, 100]
|
||||
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
|
||||
hidden = torch.randn(sum(lengths), HIDDEN, device=DEVICE_TYPE)
|
||||
first, first_questions, first_spans = schema(lengths[0])
|
||||
last, last_questions, last_spans = schema(lengths[2])
|
||||
params = [
|
||||
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
|
||||
# A prompt without a schema, like the profiling run's.
|
||||
PoolingParams(task="token_classify", use_activation=True),
|
||||
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
|
||||
]
|
||||
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
|
||||
for i, ids in enumerate(token_ids):
|
||||
padded[i, : len(ids)] = ids
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor(lengths),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=padded,
|
||||
pooling_params=params,
|
||||
pooling_states=[PoolingStates() for _ in lengths],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
torch.tensor(lengths).numpy(),
|
||||
seq_lens_cpu=torch.tensor(lengths),
|
||||
device=hidden.device,
|
||||
)
|
||||
probs, empty, logits = pooler(hidden, metadata)
|
||||
|
||||
first_hidden, _, last_hidden = torch.split(hidden, lengths)
|
||||
expected = reference_logits(
|
||||
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
|
||||
)
|
||||
expected_probs = torch.cat([q.softmax(-1) for q in expected])
|
||||
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
|
||||
assert empty.numel() == 0
|
||||
expected = reference_logits(
|
||||
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
|
||||
)
|
||||
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
|
||||
@@ -1,138 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import CLIPModel
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
HF_TEXT_PROMPTS = [
|
||||
"a photo of a stop sign",
|
||||
"a photo of a cherry blossom",
|
||||
]
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "",
|
||||
"cherry_blossom": "",
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = ["openai/clip-vit-base-patch32"]
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_cases: list[tuple[list[str], PromptImageInput]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
with vllm_runner(
|
||||
model, runner="pooling", dtype=dtype, enforce_eager=True, max_model_len=77
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.embed(input_texts, images=input_images)
|
||||
for input_texts, input_images in input_cases
|
||||
]
|
||||
|
||||
texts = [HF_TEXT_PROMPTS[0]]
|
||||
images = [input_cases[1][1][0]]
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
vllm_model.embed(texts, images=images)
|
||||
|
||||
# Should still be able to run subsequent requests
|
||||
vllm_model.embed(texts)
|
||||
vllm_model.embed([""], images=images)
|
||||
|
||||
# Mixed image+text batch must not skip the text encoder (#53091).
|
||||
mixed_outputs = vllm_outputs_per_case[2]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
|
||||
embeddings_1_lst=[mixed_outputs[0]],
|
||||
name_0="text_only",
|
||||
name_1="mixed_text",
|
||||
)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
|
||||
embeddings_1_lst=[mixed_outputs[1]],
|
||||
name_0="image_only",
|
||||
name_1="mixed_image",
|
||||
)
|
||||
empty_text = vllm_model.embed([""])
|
||||
empty_sim = torch.nn.functional.cosine_similarity(
|
||||
torch.tensor(mixed_outputs[0]),
|
||||
torch.tensor(empty_text[0]),
|
||||
dim=0,
|
||||
)
|
||||
assert empty_sim < 0.99, (
|
||||
"Mixed-batch text embedding collapsed to the empty-string vector "
|
||||
f"(cosine={empty_sim:.4f})"
|
||||
)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=CLIPModel) as hf_model:
|
||||
hf_outputs_per_case = []
|
||||
for input_texts, input_images in input_cases:
|
||||
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
|
||||
|
||||
hf_outputs = []
|
||||
for inputs in all_inputs:
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
|
||||
if "pixel_values" in inputs:
|
||||
pooled_output = hf_model.model.get_image_features(
|
||||
pixel_values=inputs.pixel_values,
|
||||
)
|
||||
else:
|
||||
pooled_output = hf_model.model.get_text_features(
|
||||
input_ids=inputs.input_ids,
|
||||
attention_mask=inputs.attention_mask,
|
||||
)
|
||||
|
||||
if not isinstance(pooled_output, torch.Tensor):
|
||||
pooled_output = pooled_output.pooler_output
|
||||
pooled_output = pooled_output.squeeze(0)
|
||||
hf_outputs.append(pooled_output.tolist())
|
||||
|
||||
hf_outputs_per_case.append(hf_outputs)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
text_images = [None] * len(HF_TEXT_PROMPTS)
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
input_cases = [
|
||||
(HF_TEXT_PROMPTS, text_images),
|
||||
(HF_IMAGE_PROMPTS, images),
|
||||
([HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]], [None, images[0]]),
|
||||
]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColModernVBERT multimodal late-interaction model.
|
||||
|
||||
ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder
|
||||
with a pixel shuffle connector and ColBERT-style 128-dim per-token
|
||||
embeddings for visual document retrieval.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
MODEL_NAME = "ModernVBERT/colmodernvbert-merged"
|
||||
COLBERT_DIM = 128
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def colmodernvbert_model(vllm_runner):
|
||||
with vllm_runner(
|
||||
MODEL_NAME,
|
||||
runner="pooling",
|
||||
dtype=DTYPE,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
yield vllm_model
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Text-only tests
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_colmodernvbert_text_token_embed(colmodernvbert_model):
|
||||
"""Text query produces per-token embeddings with shape (seq_len, 128)."""
|
||||
outputs = colmodernvbert_model.token_embed(["What is machine learning?"])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.tensor(outputs[0])
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == COLBERT_DIM
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
|
||||
def test_colmodernvbert_text_relevance_ordering(colmodernvbert_model):
|
||||
"""Relevant documents score higher than irrelevant ones."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"The weather in Paris is mild in spring.",
|
||||
]
|
||||
|
||||
scores = colmodernvbert_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
|
||||
|
||||
|
||||
def test_colmodernvbert_text_late_interaction(colmodernvbert_model):
|
||||
"""MaxSim scoring via vLLM matches manual computation."""
|
||||
query = "What is the capital of France?"
|
||||
doc = "The capital of France is Paris."
|
||||
|
||||
q_out = colmodernvbert_model.token_embed([query])
|
||||
d_out = colmodernvbert_model.token_embed([doc])
|
||||
|
||||
q_emb = torch.tensor(q_out[0])
|
||||
d_emb = torch.tensor(d_out[0])
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = colmodernvbert_model.score(query, doc)
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Image tests
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_colmodernvbert_image_token_embed(colmodernvbert_model, image_assets):
|
||||
"""Image input produces per-token embeddings including vision tokens."""
|
||||
image = image_assets[0].pil_image
|
||||
inputs = colmodernvbert_model.get_inputs(
|
||||
[""],
|
||||
images=[image],
|
||||
)
|
||||
req_outputs = colmodernvbert_model.llm.encode(
|
||||
inputs,
|
||||
pooling_task="token_embed",
|
||||
)
|
||||
outputs = [req_output.outputs.data for req_output in req_outputs]
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.tensor(outputs[0])
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == COLBERT_DIM
|
||||
# Should have at least the image tokens (64 after pixel shuffle)
|
||||
assert emb.shape[0] >= 64
|
||||
@@ -1,244 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColPali late interaction model for multi-modal retrieval.
|
||||
|
||||
ColPali is a multi-vector retrieval model based on PaliGemma backbone
|
||||
(SigLIP + Gemma) with ColBERT-style late interaction scoring (MaxSim).
|
||||
It produces per-token embeddings for both text and image inputs.
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
MODELS = [
|
||||
"vidore/colpali-v1.3-hf",
|
||||
]
|
||||
|
||||
EMBED_DIMS = {
|
||||
"vidore/colpali-v1.3-hf": 128,
|
||||
}
|
||||
|
||||
TEXT_QUERIES = [
|
||||
"What is the capital of France?",
|
||||
"Describe the contents of the document.",
|
||||
]
|
||||
|
||||
TEXT_DOCUMENTS = [
|
||||
"The capital of France is Paris.",
|
||||
"This document contains important financial data.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
GPU_MEMORY_UTILIZATION = 0.7
|
||||
|
||||
|
||||
def _make_base64_image(
|
||||
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
|
||||
) -> str:
|
||||
"""Create a small solid-color PNG image and return its base64 data URI."""
|
||||
img = Image.new("RGB", (width, height), color)
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
return f"data:image/png;base64,{b64}"
|
||||
|
||||
|
||||
def _make_image_mm_param(
|
||||
image_uri: str,
|
||||
text: str | None = None,
|
||||
) -> ScoreMultiModalParam:
|
||||
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
|
||||
content: list = [
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url={"url": image_uri},
|
||||
),
|
||||
]
|
||||
if text is not None:
|
||||
content.append(
|
||||
ChatCompletionContentPartTextParam(type="text", text=text),
|
||||
)
|
||||
return ScoreMultiModalParam(content=content)
|
||||
|
||||
|
||||
def _run_token_embed_test(
|
||||
vllm_model: VllmRunner,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""Verify per-token embedding shape and L2 normalization."""
|
||||
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.tensor(outputs[0])
|
||||
# Token embeddings should be 2D: [num_tokens, embed_dim]
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == EMBED_DIMS[model]
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
# Verify L2 normalization
|
||||
norms = torch.norm(emb, p=2, dim=-1)
|
||||
torch.testing.assert_close(
|
||||
norms,
|
||||
torch.ones_like(norms),
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def _run_late_interaction_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify MaxSim scoring matches manual computation."""
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
|
||||
|
||||
q_emb = torch.tensor(q_outputs[0])
|
||||
d_emb = torch.tensor(d_outputs[0])
|
||||
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
|
||||
def _run_relevance_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify that relevant documents score higher than irrelevant ones."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"The weather forecast shows rain tomorrow.",
|
||||
"Deep learning uses neural networks for complex tasks.",
|
||||
]
|
||||
|
||||
scores = vllm_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
|
||||
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
|
||||
|
||||
|
||||
# ── Multimodal scoring tests ────────────────────────────────
|
||||
|
||||
|
||||
def _run_multimodal_text_query_image_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score a text query against image documents via the multimodal path."""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
blue_image = _make_base64_image(64, 64, color=(0, 0, 255))
|
||||
|
||||
query = "Describe the red object"
|
||||
image_docs = [
|
||||
_make_image_mm_param(red_image),
|
||||
_make_image_mm_param(blue_image),
|
||||
]
|
||||
scores = vllm_model.llm.score(query, image_docs)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
|
||||
|
||||
def _run_multimodal_mixed_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score a text query against a mix of text and image documents."""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
|
||||
query = "What is the capital of France?"
|
||||
documents: list = [
|
||||
"The capital of France is Paris.",
|
||||
_make_image_mm_param(red_image),
|
||||
]
|
||||
|
||||
scores = vllm_model.llm.score(query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
# Text document about France should score higher than a random image
|
||||
assert scores[0].outputs.score > scores[1].outputs.score
|
||||
|
||||
|
||||
def _run_multimodal_image_query_text_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score an image query against text documents."""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
image_query = _make_image_mm_param(red_image, text="red color")
|
||||
|
||||
documents = [
|
||||
"A bright red sports car.",
|
||||
"The weather forecast shows rain tomorrow.",
|
||||
]
|
||||
|
||||
scores = vllm_model.llm.score(image_query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", [DTYPE])
|
||||
def test_colpali_default_runner(
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
|
||||
) as vllm_model:
|
||||
_run_token_embed_test(vllm_model, model)
|
||||
_run_late_interaction_test(vllm_model)
|
||||
_run_relevance_test(vllm_model)
|
||||
_run_multimodal_mixed_docs_test(vllm_model)
|
||||
_run_multimodal_image_query_text_docs_test(vllm_model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", [DTYPE])
|
||||
def test_colpali_v2_multimodal_text_query_image_docs(
|
||||
vllm_runner,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
attention_backend = "FLASH_ATTN" if current_platform.is_cuda() else None
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
|
||||
attention_backend=attention_backend,
|
||||
kernel_config={"enable_flashinfer_autotune": False},
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
_run_multimodal_text_query_image_docs_test(vllm_model)
|
||||
@@ -1,268 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColQwen3 late interaction model for multi-modal retrieval.
|
||||
|
||||
ColQwen3 is a multi-vector retrieval model based on Qwen3-VL backbone with
|
||||
ColBERT-style late interaction scoring (MaxSim). It produces per-token
|
||||
embeddings for both text and image inputs.
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
MODELS = [
|
||||
"TomoroAI/tomoro-colqwen3-embed-4b",
|
||||
"OpenSearch-AI/Ops-Colqwen3-4B",
|
||||
"nvidia/nemotron-colembed-vl-4b-v2",
|
||||
]
|
||||
|
||||
EMBED_DIMS = {
|
||||
"TomoroAI/tomoro-colqwen3-embed-4b": 320,
|
||||
"OpenSearch-AI/Ops-Colqwen3-4B": 2560,
|
||||
"nvidia/nemotron-colembed-vl-4b-v2": 2560,
|
||||
}
|
||||
|
||||
TEXT_QUERIES = [
|
||||
"What is the capital of France?",
|
||||
"Describe the contents of the document.",
|
||||
]
|
||||
|
||||
TEXT_DOCUMENTS = [
|
||||
"The capital of France is Paris.",
|
||||
"This document contains important financial data.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
# The Tomoro model needs room for its vision encoder and a 4096-token KV cache
|
||||
# on the 16 GiB devices used by the H200 MIG test lane.
|
||||
GPU_MEMORY_UTILIZATION = 0.8
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=MODELS)
|
||||
def colqwen3_model(request, vllm_runner):
|
||||
model = request.param
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=DTYPE,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=GPU_MEMORY_UTILIZATION,
|
||||
) as vllm_model:
|
||||
yield model, vllm_model
|
||||
|
||||
|
||||
def _make_base64_image(
|
||||
width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0)
|
||||
) -> str:
|
||||
"""Create a small solid-color PNG image and return its base64 data URI."""
|
||||
img = Image.new("RGB", (width, height), color)
|
||||
buf = BytesIO()
|
||||
img.save(buf, format="PNG")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
return f"data:image/png;base64,{b64}"
|
||||
|
||||
|
||||
def _make_image_mm_param(
|
||||
image_uri: str,
|
||||
text: str | None = None,
|
||||
) -> ScoreMultiModalParam:
|
||||
"""Build a ScoreMultiModalParam containing an image (and optional text)."""
|
||||
content: list = [
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url={"url": image_uri},
|
||||
),
|
||||
]
|
||||
if text is not None:
|
||||
content.append(
|
||||
ChatCompletionContentPartTextParam(type="text", text=text),
|
||||
)
|
||||
return ScoreMultiModalParam(content=content)
|
||||
|
||||
|
||||
def _make_text_mm_param(text: str) -> ScoreMultiModalParam:
|
||||
"""Build a ScoreMultiModalParam containing only text."""
|
||||
return ScoreMultiModalParam(
|
||||
content=[ChatCompletionContentPartTextParam(type="text", text=text)],
|
||||
)
|
||||
|
||||
|
||||
def _run_token_embed_test(
|
||||
vllm_model: VllmRunner,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""Verify per-token embedding shape and L2 normalization."""
|
||||
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.tensor(outputs[0])
|
||||
# Token embeddings should be 2D: [num_tokens, embed_dim]
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == EMBED_DIMS[model]
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
# Verify L2 normalization
|
||||
norms = torch.norm(emb, p=2, dim=-1)
|
||||
torch.testing.assert_close(
|
||||
norms,
|
||||
torch.ones_like(norms),
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def _run_late_interaction_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify MaxSim scoring matches manual computation."""
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
|
||||
|
||||
q_emb = torch.tensor(q_outputs[0])
|
||||
d_emb = torch.tensor(d_outputs[0])
|
||||
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
|
||||
def _run_relevance_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify that relevant documents score higher than irrelevant ones."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"The weather forecast shows rain tomorrow.",
|
||||
"Deep learning uses neural networks for complex tasks.",
|
||||
]
|
||||
|
||||
scores = vllm_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
|
||||
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
|
||||
|
||||
|
||||
def test_colqwen3_token_embed(colqwen3_model) -> None:
|
||||
model, vllm_model = colqwen3_model
|
||||
_run_token_embed_test(vllm_model, model)
|
||||
|
||||
|
||||
def test_colqwen3_late_interaction_scoring(colqwen3_model) -> None:
|
||||
_, vllm_model = colqwen3_model
|
||||
_run_late_interaction_test(vllm_model)
|
||||
|
||||
|
||||
def test_colqwen3_relevance_ordering(colqwen3_model) -> None:
|
||||
_, vllm_model = colqwen3_model
|
||||
_run_relevance_test(vllm_model)
|
||||
|
||||
|
||||
# ── Multimodal scoring tests ────────────────────────────────
|
||||
|
||||
|
||||
def _run_multimodal_text_query_image_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score a text query against image documents via the multimodal path.
|
||||
|
||||
Verifies that score_data_to_prompts correctly handles image content
|
||||
and produces valid MaxSim scores.
|
||||
"""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
blue_image = _make_base64_image(64, 64, color=(0, 0, 255))
|
||||
|
||||
query = "Describe the red object"
|
||||
image_docs = [
|
||||
_make_image_mm_param(red_image),
|
||||
_make_image_mm_param(blue_image),
|
||||
]
|
||||
|
||||
scores = vllm_model.llm.score(query, image_docs)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
|
||||
|
||||
def _run_multimodal_mixed_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score a text query against a mix of text and image documents.
|
||||
|
||||
Ensures the late-interaction path handles heterogeneous document
|
||||
types (plain strings alongside ScoreMultiModalParam images) in
|
||||
a single call.
|
||||
"""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
|
||||
query = "What is the capital of France?"
|
||||
documents: list = [
|
||||
"The capital of France is Paris.",
|
||||
_make_image_mm_param(red_image),
|
||||
]
|
||||
|
||||
scores = vllm_model.llm.score(query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
# Text document about France should score higher than a random image
|
||||
assert scores[0].outputs.score > scores[1].outputs.score
|
||||
|
||||
|
||||
def _run_multimodal_image_query_text_docs_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Score an image query against text documents.
|
||||
|
||||
Verifies the reverse direction: multimodal query with text-only
|
||||
documents through the late-interaction scoring path.
|
||||
"""
|
||||
red_image = _make_base64_image(64, 64, color=(255, 0, 0))
|
||||
image_query = _make_image_mm_param(red_image, text="red color")
|
||||
|
||||
documents = [
|
||||
"A bright red sports car.",
|
||||
"The weather forecast shows rain tomorrow.",
|
||||
]
|
||||
|
||||
scores = vllm_model.llm.score(image_query, documents)
|
||||
|
||||
assert len(scores) == 2
|
||||
for s in scores:
|
||||
assert isinstance(s.outputs.score, float)
|
||||
|
||||
|
||||
def test_colqwen3_multimodal_text_query_image_docs(colqwen3_model) -> None:
|
||||
_, vllm_model = colqwen3_model
|
||||
_run_multimodal_text_query_image_docs_test(vllm_model)
|
||||
|
||||
|
||||
def test_colqwen3_multimodal_mixed_docs(colqwen3_model) -> None:
|
||||
_, vllm_model = colqwen3_model
|
||||
_run_multimodal_mixed_docs_test(vllm_model)
|
||||
|
||||
|
||||
def test_colqwen3_multimodal_image_query_text_docs(colqwen3_model) -> None:
|
||||
_, vllm_model = colqwen3_model
|
||||
_run_multimodal_image_query_text_docs_test(vllm_model)
|
||||
@@ -1,236 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ColQwen3.5 late interaction model for multi-modal retrieval.
|
||||
|
||||
ColQwen3.5 is a multi-vector retrieval model based on Qwen3.5 backbone with
|
||||
ColBERT-style late interaction scoring (MaxSim). It produces per-token
|
||||
embeddings for both text and image inputs.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
MODELS = [
|
||||
"athrael-soju/colqwen3.5-4.5B-v3",
|
||||
]
|
||||
|
||||
EMBED_DIMS = {
|
||||
"athrael-soju/colqwen3.5-4.5B-v3": 320,
|
||||
}
|
||||
|
||||
TEXT_QUERIES = [
|
||||
"What is the capital of France?",
|
||||
"Describe the contents of the document.",
|
||||
]
|
||||
|
||||
TEXT_DOCUMENTS = [
|
||||
"The capital of France is Paris.",
|
||||
"This document contains important financial data.",
|
||||
]
|
||||
|
||||
DTYPE = "half"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", params=MODELS)
|
||||
def colqwen3_5_model(request, vllm_runner):
|
||||
model = request.param
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=DTYPE,
|
||||
max_model_len=4096,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
yield model, vllm_model
|
||||
|
||||
|
||||
def _run_token_embed_test(
|
||||
vllm_model: VllmRunner,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""Verify per-token embedding shape and L2 normalization."""
|
||||
outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
|
||||
assert len(outputs) == 1
|
||||
emb = torch.tensor(outputs[0])
|
||||
# Token embeddings should be 2D: [num_tokens, embed_dim]
|
||||
assert emb.dim() == 2
|
||||
assert emb.shape[1] == EMBED_DIMS[model]
|
||||
assert emb.shape[0] > 1
|
||||
|
||||
# Verify L2 normalization
|
||||
norms = torch.norm(emb, p=2, dim=-1)
|
||||
torch.testing.assert_close(
|
||||
norms,
|
||||
torch.ones_like(norms),
|
||||
rtol=1e-2,
|
||||
atol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
def _run_late_interaction_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify MaxSim scoring matches manual computation."""
|
||||
from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
|
||||
|
||||
q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]])
|
||||
d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]])
|
||||
|
||||
q_emb = torch.tensor(q_outputs[0])
|
||||
d_emb = torch.tensor(d_outputs[0])
|
||||
|
||||
manual_score = compute_maxsim_score(q_emb, d_emb).item()
|
||||
|
||||
vllm_scores = vllm_model.score(TEXT_QUERIES[0], TEXT_DOCUMENTS[0])
|
||||
|
||||
assert len(vllm_scores) == 1
|
||||
assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
|
||||
|
||||
|
||||
def _run_relevance_test(
|
||||
vllm_model: VllmRunner,
|
||||
) -> None:
|
||||
"""Verify that relevant documents score higher than irrelevant ones."""
|
||||
query = "What is machine learning?"
|
||||
documents = [
|
||||
"Machine learning is a subset of artificial intelligence.",
|
||||
"The weather forecast shows rain tomorrow.",
|
||||
"Deep learning uses neural networks for complex tasks.",
|
||||
]
|
||||
|
||||
scores = vllm_model.score(query, documents)
|
||||
|
||||
assert len(scores) == 3
|
||||
assert scores[0] > scores[1], "ML doc should score higher than weather doc"
|
||||
assert scores[2] > scores[1], "DL doc should score higher than weather doc"
|
||||
|
||||
|
||||
def test_colqwen3_5_token_embed(colqwen3_5_model) -> None:
|
||||
model, vllm_model = colqwen3_5_model
|
||||
_run_token_embed_test(vllm_model, model)
|
||||
|
||||
|
||||
def test_colqwen3_5_late_interaction_scoring(colqwen3_5_model) -> None:
|
||||
_, vllm_model = colqwen3_5_model
|
||||
_run_late_interaction_test(vllm_model)
|
||||
|
||||
|
||||
def test_colqwen3_5_relevance_ordering(colqwen3_5_model) -> None:
|
||||
_, vllm_model = colqwen3_5_model
|
||||
_run_relevance_test(vllm_model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("contract", "expected_is_causal"),
|
||||
[("causal", True), ("bidirectional", False)],
|
||||
)
|
||||
def test_colqwen3_5_config_applies_declared_attention_contract(
|
||||
contract: str,
|
||||
expected_is_causal: bool,
|
||||
) -> None:
|
||||
from vllm.model_executor.models.config import (
|
||||
MODELS_CONFIG_MAP,
|
||||
ColQwen3_5Config,
|
||||
)
|
||||
|
||||
assert MODELS_CONFIG_MAP["ColQwen3_5"] is ColQwen3_5Config
|
||||
|
||||
hf_config = SimpleNamespace(retrieval_attention_contract=contract)
|
||||
text_config = SimpleNamespace()
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
assert hf_config.is_causal is expected_is_causal
|
||||
assert text_config.is_causal is expected_is_causal
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hf_config",
|
||||
[
|
||||
SimpleNamespace(),
|
||||
SimpleNamespace(retrieval_attention_contract="unsupported"),
|
||||
SimpleNamespace(
|
||||
retrieval_attention_contract="causal",
|
||||
text_config=SimpleNamespace(retrieval_attention_contract="bidirectional"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_colqwen3_5_config_rejects_invalid_attention_contract(hf_config) -> None:
|
||||
from vllm.model_executor.models.config import ColQwen3_5Config
|
||||
|
||||
text_config = getattr(hf_config, "text_config", SimpleNamespace())
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
with pytest.raises(ValueError, match="retrieval_attention_contract"):
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
|
||||
|
||||
def test_colqwen3_5_bidirectional_contract_builds_encoder_only_attention(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.models import qwen3_next
|
||||
from vllm.model_executor.models.config import ColQwen3_5Config
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
hf_config = SimpleNamespace(retrieval_attention_contract="bidirectional")
|
||||
text_config = SimpleNamespace(
|
||||
hidden_size=256,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
max_position_embeddings=4096,
|
||||
rope_parameters={},
|
||||
rms_norm_eps=1e-6,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakeAttention(torch.nn.Module):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__()
|
||||
captured["attn_type"] = kwargs["attn_type"]
|
||||
|
||||
monkeypatch.setattr(qwen3_next, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "QKVParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "RowParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next,
|
||||
"get_rope",
|
||||
lambda *args, **kwargs: SimpleNamespace(is_neox_style=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "Qwen3NextRMSNorm", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(qwen3_next, "Attention", FakeAttention)
|
||||
|
||||
qwen3_next.Qwen3NextAttention(text_config)
|
||||
|
||||
assert captured["attn_type"] is AttentionType.ENCODER_ONLY
|
||||
|
||||
|
||||
def test_colqwen3_5_encoder_only_attention_has_no_kv_cache_spec() -> None:
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
attention = SimpleNamespace(attn_type=AttentionType.ENCODER_ONLY)
|
||||
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=16))
|
||||
|
||||
assert Attention.get_kv_cache_spec(attention, vllm_config) is None
|
||||
@@ -1,87 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Decision 3.0 against the checkpoint's own runtime (`trust_remote_code`).
|
||||
|
||||
The runtime reads a question's options from the 255 code logits of its readout
|
||||
at the last prompt token. vLLM's classify head must give the same code logits
|
||||
for the same prompt tokens, and for an image prompt that vLLM processes itself.
|
||||
d3-edge has an irregular layer layout; d3-nano ships its backbone as a single
|
||||
weight file without an index, next to the readout file.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODELS = ["vllm-sr/d3-edge", "vllm-sr/d3-nano"]
|
||||
STATE = (
|
||||
"The order arrived damaged yesterday. The customer has a receipt and asks "
|
||||
"for a replacement today."
|
||||
)
|
||||
QUESTIONS = {
|
||||
"route": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team should handle this request?",
|
||||
"criteria": {
|
||||
"returns": "Refunds, replacements and damaged deliveries",
|
||||
"billing": "Payments, invoices and charges",
|
||||
"technical": "Product setup and faults",
|
||||
},
|
||||
},
|
||||
"receipt": {"type": "noul", "instructions": "Does the customer have a receipt?"},
|
||||
"urgency": {
|
||||
"type": "score",
|
||||
"instructions": "How urgent is this request?",
|
||||
"criteria": ["Routine", "Soon", "Today"],
|
||||
},
|
||||
"department": {
|
||||
"type": "choice",
|
||||
"criteria": {f"department {i}": None for i in range(40)},
|
||||
},
|
||||
}
|
||||
IMAGE_QUESTIONS = {"photo": {"type": "noul", "instructions": "Is this a photograph?"}}
|
||||
|
||||
|
||||
def option_probs(logits: torch.Tensor, count: int) -> torch.Tensor:
|
||||
return torch.softmax(logits[:count].float(), dim=-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_d3_matches_the_checkpoint_runtime(vllm_runner, model: str) -> None:
|
||||
reference = AutoModel.from_pretrained(model, trust_remote_code=True)
|
||||
runtime = reference.runtime
|
||||
text = runtime.prepare(STATE, QUESTIONS)
|
||||
keys = text.runnable
|
||||
counts = [len(text.questions[key].keys) for key in keys]
|
||||
sequences = [text.sequences[key] for key in keys]
|
||||
ref_logits = list(runtime.logits(sequences, counts).cpu())
|
||||
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
visual = runtime.prepare("Describe the attached image.", IMAGE_QUESTIONS, [image])
|
||||
(image_key,) = visual.runnable
|
||||
image_prompt = visual.texts[image_key]
|
||||
ref_logits += list(
|
||||
runtime.image_logits(
|
||||
[image_prompt], visual.images, [2], visual.lengths[image_key]
|
||||
).cpu()
|
||||
)
|
||||
counts.append(2)
|
||||
del reference, runtime
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
with vllm_runner(model, runner="pooling", max_model_len=4096) as vllm_model:
|
||||
outputs = vllm_model.llm.classify(
|
||||
[{"prompt_token_ids": ids} for ids in sequences]
|
||||
+ [{"prompt": image_prompt, "multi_modal_data": {"image": image}}],
|
||||
use_tqdm=False,
|
||||
)
|
||||
# The head returns raw code logits by default.
|
||||
vllm_logits = [torch.tensor(output.outputs.probs) for output in outputs]
|
||||
assert all(len(logits) == 255 for logits in vllm_logits)
|
||||
|
||||
for ref, ours, count in zip(ref_logits, vllm_logits, counts):
|
||||
ref_probs, our_probs = option_probs(ref, count), option_probs(ours, count)
|
||||
torch.testing.assert_close(our_probs, ref_probs, atol=0.02, rtol=0)
|
||||
@@ -1,215 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from PIL import Image
|
||||
from transformers import Qwen2VLForConditionalGeneration
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
HF_TEXT_PROMPTS = [
|
||||
# T -> X
|
||||
(
|
||||
"Query: Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501,
|
||||
Image.new("RGB", (56, 56)),
|
||||
),
|
||||
# T -> X
|
||||
(
|
||||
"Query: Retrieve an image of this caption: cherry blossom",
|
||||
Image.new("RGB", (56, 56)),
|
||||
),
|
||||
]
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "What is shown in this image?",
|
||||
"cherry_blossom": "What is shown in this image?",
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = ["MrLight/dse-qwen2-2b-mrl-v1"]
|
||||
|
||||
|
||||
def get_messages(image: Image.Image, text: str, embed_text: bool):
|
||||
# assert False, 'remember to use outer [] as required'
|
||||
if embed_text:
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"image": Image.new("RGB", (56, 56)),
|
||||
"resized_height": 1,
|
||||
"resized_width": 1,
|
||||
}, # need a dummy image here for an easier process.
|
||||
{"type": "text", "text": text},
|
||||
],
|
||||
}
|
||||
]
|
||||
else:
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image", "image": image},
|
||||
{"type": "text", "text": text},
|
||||
],
|
||||
}
|
||||
]
|
||||
return messages
|
||||
|
||||
|
||||
def apply_chat_template_and_add_eos(
|
||||
messages: list[dict],
|
||||
apply_chat_template_fn: Callable,
|
||||
):
|
||||
prompt = (
|
||||
apply_chat_template_fn(messages, tokenize=False, add_generation_prompt=True)
|
||||
+ "<|endoftext|>"
|
||||
)
|
||||
return prompt
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_texts: list[str],
|
||||
input_images: PromptImageInput,
|
||||
embed_texts: list[bool],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""SET PYTHONPATH."""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
with vllm_runner(
|
||||
model, runner="pooling", dtype=dtype, enforce_eager=True, max_model_len=8192
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
texts = [
|
||||
# this is necessary because vllm_model.embed will not apply any
|
||||
# templating to the prompt, and therefore lacks an image_pad
|
||||
# token unless one is inserted beforehand (the (28,28) image
|
||||
# above is converted to an image pad token by the chat template).
|
||||
apply_chat_template_and_add_eos(
|
||||
get_messages(image, text, False),
|
||||
apply_chat_template_fn=tokenizer.apply_chat_template,
|
||||
)
|
||||
for text, image in zip(input_texts, input_images)
|
||||
# vllm will replace the pad token with the actual image,
|
||||
# which may be a placeholder image, later.
|
||||
]
|
||||
vllm_outputs = vllm_model.embed(texts, images=input_images)
|
||||
|
||||
hf_outputs = []
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=Qwen2VLForConditionalGeneration
|
||||
) as hf_model:
|
||||
prompts = []
|
||||
for text, image, embed_text in zip(input_texts, input_images, embed_texts):
|
||||
# dse requires non-standard input processing
|
||||
# because it needs an image_pad token
|
||||
messages = get_messages(image, text, embed_text)
|
||||
prompt = apply_chat_template_and_add_eos(
|
||||
messages, hf_model.processor.apply_chat_template
|
||||
)
|
||||
|
||||
prompts.append(prompt)
|
||||
|
||||
all_inputs = hf_model.get_inputs(
|
||||
prompts=prompts,
|
||||
images=input_images,
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
all_outputs = []
|
||||
for inputs in all_inputs:
|
||||
inputs = hf_model.model.prepare_inputs_for_generation(
|
||||
**inputs,
|
||||
cache_position=torch.arange(1), # 1 for batch size
|
||||
use_cache=False,
|
||||
)
|
||||
outputs = hf_model.model(
|
||||
**hf_model.wrap_device(inputs),
|
||||
return_dict=True,
|
||||
output_hidden_states=True,
|
||||
)
|
||||
pooled_output = F.normalize(
|
||||
outputs.hidden_states[-1][0, -1], p=2, dim=-1
|
||||
)
|
||||
|
||||
all_outputs.append(pooled_output.tolist())
|
||||
|
||||
hf_outputs = all_outputs
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_models_text(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [
|
||||
(text, image_placeholder) for text, image_placeholder in HF_TEXT_PROMPTS
|
||||
]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
embed_texts = [True] * len(input_texts)
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images, # type: ignore
|
||||
embed_texts,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=48)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_models_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [
|
||||
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
|
||||
]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
embed_texts = [False] * len(input_texts)
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images,
|
||||
embed_texts,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.clip import (
|
||||
dual_encoder_has_text_tokens,
|
||||
merge_dual_encoder_text_and_vision,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def test_text_only_batch_runs_text_encoder():
|
||||
assert dual_encoder_has_text_tokens(False, None)
|
||||
assert dual_encoder_has_text_tokens(False, torch.zeros(4, dtype=torch.bool))
|
||||
|
||||
|
||||
def test_vision_only_batch_skips_text_encoder():
|
||||
is_multimodal = torch.ones(8, dtype=torch.bool)
|
||||
assert not dual_encoder_has_text_tokens(True, is_multimodal)
|
||||
|
||||
|
||||
def test_mixed_batch_runs_text_encoder():
|
||||
# Image tokens followed by a text sequence: the old batch-wide flag
|
||||
# treated this as vision-only and skipped the text encoder (#53091).
|
||||
is_multimodal = torch.tensor([True, True, True, False, False, False])
|
||||
assert dual_encoder_has_text_tokens(True, is_multimodal)
|
||||
|
||||
|
||||
def test_missing_token_mask_with_mm_embeddings_is_vision_only():
|
||||
assert not dual_encoder_has_text_tokens(True, None)
|
||||
|
||||
|
||||
def test_merge_keeps_vision_on_mm_tokens():
|
||||
text = torch.zeros(4, 2)
|
||||
vision = torch.ones(4, 2)
|
||||
is_multimodal = torch.tensor([True, True, False, False])
|
||||
out = merge_dual_encoder_text_and_vision(text, vision, is_multimodal)
|
||||
assert torch.equal(out[:2], vision[:2])
|
||||
assert torch.equal(out[2:], text[2:])
|
||||
@@ -1,660 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import os
|
||||
import textwrap
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
MODEL = os.environ.get("EG2_MODEL_PATH")
|
||||
needs_model = pytest.mark.skipif(not MODEL, reason="EG2_MODEL_PATH not set")
|
||||
TEXT_COS, MM_COS = 0.999, 0.999
|
||||
|
||||
|
||||
def _cos(a, b):
|
||||
return torch.nn.functional.cosine_similarity(
|
||||
torch.tensor(a, dtype=torch.float32),
|
||||
torch.tensor(b, dtype=torch.float32),
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
|
||||
def _ensure_synthetic_video(
|
||||
path: str, duration: float, fps: float, width: int, height: int
|
||||
) -> str:
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
fourcc_fn = getattr(cv2, "VideoWriter_fourcc", cv2.VideoWriter.fourcc)
|
||||
fourcc = fourcc_fn(*"mp4v")
|
||||
num_frames = int(duration * fps)
|
||||
writer = cv2.VideoWriter(path, fourcc, fps, (width, height))
|
||||
y_coords, x_coords = np.mgrid[0:height, 0:width]
|
||||
for i in range(num_frames):
|
||||
# Spatially varying color pattern that evolves dynamically over frames
|
||||
frame: np.ndarray = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
frame[..., 0] = ((x_coords * 255 // max(1, width)) + i * 17) % 256
|
||||
frame[..., 1] = ((y_coords * 255 // max(1, height)) + i * 29) % 256
|
||||
frame[..., 2] = (x_coords + y_coords + i * 13) % 256
|
||||
writer.write(frame)
|
||||
writer.release()
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def st_model():
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
return SentenceTransformer(
|
||||
MODEL, device="cuda", model_kwargs={"torch_dtype": torch.bfloat16}
|
||||
)
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_structure(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
|
||||
def probe(model):
|
||||
lm = model.language_model
|
||||
win = [layer.self_attn.attn.impl.sliding_window for layer in lm.layers]
|
||||
return win, model.config.text_config.sliding_window
|
||||
|
||||
win, cfg_w = vm.apply_model(probe)[0]
|
||||
assert (cfg_w, cfg_w) in win # (512, 512) == W-1 with W = 513
|
||||
|
||||
|
||||
@needs_model
|
||||
@pytest.mark.parametrize(
|
||||
"prompt",
|
||||
[
|
||||
"short query",
|
||||
"task: search result | query: what is vllm",
|
||||
"word " * 700, # > 512: crosses sliding window
|
||||
"word " * 1500, # > 1024
|
||||
],
|
||||
)
|
||||
def test_text_parity(vllm_runner, st_model, prompt):
|
||||
ref = st_model.encode([prompt], normalize_embeddings=True)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed([prompt])
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= TEXT_COS, (
|
||||
f"Text cosine {sim} < {TEXT_COS} for prompt len {len(prompt)}"
|
||||
)
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_mm_recipe(vllm_runner, st_model):
|
||||
from PIL import Image
|
||||
|
||||
img = Image.new("RGB", (224, 224), color=(255, 0, 0))
|
||||
prompt = "Describe this image: <|image|>"
|
||||
|
||||
# SentenceTransformer multimodal encode
|
||||
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Image cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_natural_image_parity(vllm_runner, st_model):
|
||||
from PIL import Image
|
||||
|
||||
img_path = "/projects/gemma4-vllm/images/cat.jpg"
|
||||
if not os.path.exists(img_path):
|
||||
pytest.skip(f"Test asset {img_path} not found")
|
||||
img = Image.open(img_path)
|
||||
prompt = "What is this image? <|image|>"
|
||||
ref = st_model.encode([{"image": img, "text": prompt}], normalize_embeddings=True)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed([{"prompt": prompt, "multi_modal_data": {"image": img}}])
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Natural image cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_audio_parity(vllm_runner, st_model):
|
||||
import torchaudio
|
||||
|
||||
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
|
||||
if not os.path.exists(audio_path):
|
||||
pytest.skip(f"Test asset {audio_path} not found")
|
||||
wav, sr = torchaudio.load(audio_path)
|
||||
mono_wav = wav.mean(dim=0).numpy()
|
||||
prompt = "Describe this audio: <|audio|>"
|
||||
ref = st_model.encode(
|
||||
[{"audio": audio_path, "text": prompt}], normalize_embeddings=True
|
||||
)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[{"prompt": prompt, "multi_modal_data": {"audio": (mono_wav, sr)}}]
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Audio cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_mixed_parity(vllm_runner, st_model):
|
||||
import torchaudio
|
||||
from PIL import Image
|
||||
|
||||
img_path = "/projects/gemma4-vllm/images/cat.jpg"
|
||||
audio_path = "/projects/gemma4-vllm/audio/tellMeAboutTheSun.wav"
|
||||
if not os.path.exists(img_path) or not os.path.exists(audio_path):
|
||||
pytest.skip("Test assets for mixed parity not found")
|
||||
img = Image.open(img_path)
|
||||
wav, sr = torchaudio.load(audio_path)
|
||||
mono_wav = wav.mean(dim=0).numpy()
|
||||
prompt = "Describe image <|image|> and audio <|audio|>"
|
||||
ref = st_model.encode(
|
||||
[{"image": img, "audio": audio_path, "text": prompt}],
|
||||
normalize_embeddings=True,
|
||||
)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"image": img,
|
||||
"audio": (mono_wav, sr),
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Mixed cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_video_parity_uniform_overflow(vllm_runner, st_model):
|
||||
from transformers.video_utils import load_video
|
||||
|
||||
video_path = _ensure_synthetic_video(
|
||||
"/tmp/test_40s.mp4", duration=40.0, fps=10.0, width=160, height=120
|
||||
)
|
||||
frames, meta = load_video(video_path, backend="opencv")
|
||||
prompt = "Describe this video: <|video|>"
|
||||
ref = st_model.encode(
|
||||
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||
)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"40s video cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_video_parity_portrait(vllm_runner, st_model):
|
||||
from transformers.video_utils import load_video
|
||||
|
||||
video_path = _ensure_synthetic_video(
|
||||
"/tmp/test_portrait.mp4", duration=3.0, fps=10.0, width=120, height=160
|
||||
)
|
||||
frames, meta = load_video(video_path, backend="opencv")
|
||||
prompt = "Describe this video: <|video|>"
|
||||
ref = st_model.encode(
|
||||
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||
)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Portrait video cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_tower_skip(vllm_runner, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
def nparams(m):
|
||||
return sum(p.numel() for p in m.parameters()) / 1e6
|
||||
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
runner="pooling",
|
||||
max_model_len=2048,
|
||||
limit_mm_per_prompt={"image": 0, "audio": 0, "video": 0},
|
||||
) as vm:
|
||||
p_text = vm.apply_model(nparams)[0]
|
||||
assert abs(p_text - 271.00) <= 0.05, (
|
||||
f"Expected 271.00M text params, got {p_text}M"
|
||||
)
|
||||
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
|
||||
p_full = vm.apply_model(nparams)[0]
|
||||
assert abs(p_full - 744.37) <= 0.05, (
|
||||
f"Expected 744.37M full params, got {p_full}M"
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_contract():
|
||||
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ForConditionalGeneration as G,
|
||||
)
|
||||
|
||||
borrowed_methods = [
|
||||
"_parse_and_validate_multimodal_inputs",
|
||||
"_parse_and_validate_image_input",
|
||||
"_parse_and_validate_video_input",
|
||||
"_parse_and_validate_audio_input",
|
||||
"_process_image_input",
|
||||
"_process_video_input",
|
||||
"_process_audio_input",
|
||||
"embed_multimodal",
|
||||
"_encoder_chunk",
|
||||
]
|
||||
|
||||
for name in borrowed_methods:
|
||||
# Check method identity (L-F6 + T-F4)
|
||||
assert getattr(EmbeddingGemma2Model, name) is getattr(G, name)
|
||||
|
||||
# AST checks: verify no zero-arg super() calls in borrowed methods
|
||||
func = getattr(EmbeddingGemma2Model, name)
|
||||
src = textwrap.dedent(inspect.getsource(func))
|
||||
tree = ast.parse(src)
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id == "super"
|
||||
):
|
||||
assert len(node.args) > 0, (
|
||||
f"Zero-arg super() call found in borrowed method {name}"
|
||||
)
|
||||
|
||||
# AST check: all self.<attr> non-callable accesses are assigned in __init__
|
||||
init_src = inspect.getsource(EmbeddingGemma2Model.__init__)
|
||||
for name in borrowed_methods:
|
||||
func = getattr(EmbeddingGemma2Model, name)
|
||||
src = textwrap.dedent(inspect.getsource(func))
|
||||
tree = ast.parse(src)
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
):
|
||||
attr = node.attr
|
||||
assert hasattr(EmbeddingGemma2Model, attr) or attr in init_src, (
|
||||
f"Attribute {attr} accessed on self in {name} is neither "
|
||||
f"defined on class nor in __init__"
|
||||
)
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_instance_attribute_access(vllm_runner):
|
||||
"""Check self.<attr> accesses on a real model instance."""
|
||||
from vllm.model_executor.models.embedding_gemma2 import EmbeddingGemma2Model
|
||||
|
||||
borrowed_methods = [
|
||||
"_parse_and_validate_multimodal_inputs",
|
||||
"_parse_and_validate_image_input",
|
||||
"_parse_and_validate_video_input",
|
||||
"_parse_and_validate_audio_input",
|
||||
"_process_image_input",
|
||||
"_process_video_input",
|
||||
"_process_audio_input",
|
||||
"embed_multimodal",
|
||||
"_encoder_chunk",
|
||||
]
|
||||
attrs_accessed = set()
|
||||
for name in borrowed_methods:
|
||||
func = getattr(EmbeddingGemma2Model, name)
|
||||
src = textwrap.dedent(inspect.getsource(func))
|
||||
tree = ast.parse(src)
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
):
|
||||
attrs_accessed.add(node.attr)
|
||||
|
||||
def check_attrs(model):
|
||||
for attr in attrs_accessed:
|
||||
assert hasattr(model, attr), (
|
||||
f"Real instance of EmbeddingGemma2Model is missing attribute '{attr}' "
|
||||
f"accessed by borrowed method"
|
||||
)
|
||||
return True
|
||||
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=2048) as vm:
|
||||
res = vm.apply_model(check_attrs)
|
||||
assert res[0] is True
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_video_loader_real_video(vllm_runner, st_model):
|
||||
"""Decode a real MP4 file through EmbeddingGemma2VideoBackend.load_bytes
|
||||
|
||||
and VideoMediaIO / MediaConnector with default num_frames=32.
|
||||
"""
|
||||
from vllm.multimodal.media import MediaConnector
|
||||
from vllm.multimodal.media.video import VideoMediaIO
|
||||
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
||||
|
||||
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
|
||||
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||
with open(video_path, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
# Direct backend load
|
||||
frames, meta = backend_cls.load_bytes(data)
|
||||
|
||||
# VideoMediaIO with default num_frames=32 (VideoMediaIO passes num_frames=32)
|
||||
video_io = VideoMediaIO(image_io=None, video_backend="embedding_gemma2")
|
||||
loaded = video_io.load_bytes(data)
|
||||
frames_io, meta_io = loaded.media
|
||||
assert len(frames_io) == len(frames)
|
||||
assert meta_io.get("do_sample_frames") is False
|
||||
|
||||
# MediaConnector with video_processor="EmbeddingGemma2VideoProcessor"
|
||||
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
|
||||
fetched = connector.fetch_video(
|
||||
f"file://{video_path}",
|
||||
video_processor="EmbeddingGemma2VideoProcessor",
|
||||
)
|
||||
frames_conn, meta_conn = fetched.media
|
||||
assert len(frames_conn) == 32
|
||||
assert meta_conn.get("do_sample_frames") is False
|
||||
|
||||
prompt = "Describe this video: <|video|>"
|
||||
ref = st_model.encode(
|
||||
[{"video": video_path, "text": prompt}], normalize_embeddings=True
|
||||
)
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"video": (frames_conn, meta_conn)},
|
||||
}
|
||||
]
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Real video loader cosine {sim} < {MM_COS}"
|
||||
|
||||
|
||||
@needs_model
|
||||
def test_video_loader_duplicate_expansion_e2e(vllm_runner, st_model):
|
||||
"""End-to-end duplicate expansion test with low-fps clip."""
|
||||
import numpy as np
|
||||
|
||||
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
||||
|
||||
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||
# 4s clip at 0.5 fps = 2 unique frames
|
||||
lowfps_path = "/tmp/test_lowfps_05.mp4"
|
||||
_ensure_synthetic_video(lowfps_path, duration=4.0, fps=0.5, width=64, height=48)
|
||||
with open(lowfps_path, "rb") as f:
|
||||
data = f.read()
|
||||
# Default fps=1.0 -> 4 frames expanding 2 unique frames to [0, 0, 1, 1]
|
||||
frames, meta = backend_cls.load_bytes(data)
|
||||
assert frames.shape[0] == 4
|
||||
# Compare returned frames directly, not only indices
|
||||
assert np.array_equal(frames[0], frames[1])
|
||||
assert not np.array_equal(frames[0], frames[2])
|
||||
assert np.array_equal(frames[2], frames[3])
|
||||
|
||||
prompt = "Describe this video: <|video|>"
|
||||
with vllm_runner(MODEL, runner="pooling", max_model_len=8192) as vm:
|
||||
out = vm.embed(
|
||||
[{"prompt": prompt, "multi_modal_data": {"video": (frames, meta)}}]
|
||||
)
|
||||
ref = st_model.encode(
|
||||
[{"video": lowfps_path, "text": prompt}], normalize_embeddings=True
|
||||
)
|
||||
sim = _cos(out, ref).min().item()
|
||||
assert sim >= MM_COS, f"Duplicate expansion cosine {sim} < {MM_COS}"
|
||||
assert len(out) == 1
|
||||
assert len(out[0]) == 768
|
||||
|
||||
|
||||
def test_video_loader_indices():
|
||||
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
|
||||
EmbeddingGemma2VideoProcessor,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
)
|
||||
|
||||
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||
hf_proc = EmbeddingGemma2VideoProcessor()
|
||||
|
||||
# Case 1: clip shorter than 32s (e.g. 5s @ 24fps = 120 frames)
|
||||
src_5s = VideoSourceMetadata(total_frames_num=120, original_fps=24.0, duration=5.0)
|
||||
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||
vllm_indices = backend_cls.compute_frames_index_to_sample(src_5s, tgt)
|
||||
hf_meta_5s = VideoMetadata(fps=24.0, duration=5.0, total_num_frames=120)
|
||||
hf_indices_5s = hf_proc.sample_frames(
|
||||
hf_meta_5s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||
).tolist()
|
||||
assert vllm_indices == hf_indices_5s
|
||||
|
||||
# Case 2: clip shorter than 1s (e.g. 0.5s @ 24fps = 12 frames)
|
||||
src_sub1s = VideoSourceMetadata(
|
||||
total_frames_num=12, original_fps=24.0, duration=0.5
|
||||
)
|
||||
vllm_sub1s = backend_cls.compute_frames_index_to_sample(src_sub1s, tgt)
|
||||
hf_meta_sub1s = VideoMetadata(fps=24.0, duration=0.5, total_num_frames=12)
|
||||
hf_indices_sub1s = hf_proc.sample_frames(
|
||||
hf_meta_sub1s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||
).tolist()
|
||||
assert vllm_sub1s == hf_indices_sub1s
|
||||
|
||||
# Case 3: clip longer than 32s (e.g. 40s @ 10fps = 400 frames)
|
||||
# -> uniform overflow cap 32 frames
|
||||
src_40s = VideoSourceMetadata(
|
||||
total_frames_num=400, original_fps=10.0, duration=40.0
|
||||
)
|
||||
vllm_40s = backend_cls.compute_frames_index_to_sample(src_40s, tgt)
|
||||
hf_meta_40s = VideoMetadata(fps=10.0, duration=40.0, total_num_frames=400)
|
||||
hf_indices_40s = hf_proc.sample_frames(
|
||||
hf_meta_40s, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||
).tolist()
|
||||
assert vllm_40s == hf_indices_40s
|
||||
assert len(vllm_40s) == 32
|
||||
|
||||
# Case 4: default target fps (target.fps <= 0 defaults to 1.0)
|
||||
tgt_default_fps = VideoTargetMetadata(num_frames=-1, fps=-1.0, max_duration=300)
|
||||
vllm_def = backend_cls.compute_frames_index_to_sample(src_5s, tgt_default_fps)
|
||||
assert vllm_def == hf_indices_5s
|
||||
|
||||
# Case 5 (NEW-2): 60 fps / 300 s (18000 frames) vs HF sample_frames
|
||||
src_300s = VideoSourceMetadata(
|
||||
total_frames_num=18000, original_fps=60.0, duration=300.0
|
||||
)
|
||||
v_300s = backend_cls.compute_frames_index_to_sample(
|
||||
src_300s, VideoTargetMetadata(num_frames=-1, fps=60.0, max_duration=300)
|
||||
)
|
||||
h_300s = hf_proc.sample_frames(
|
||||
VideoMetadata(fps=60.0, duration=300.0, total_num_frames=18000),
|
||||
fps=60.0,
|
||||
max_frames=32,
|
||||
overflow_strategy="uniform",
|
||||
).tolist()
|
||||
assert v_300s == h_300s, f"60fps/300s mismatch: {v_300s[-3:]} vs {h_300s[-3:]}"
|
||||
|
||||
# Case 6 (NEW-2): 1 fps / 4 h (432000 frames @ 30 native fps) vs HF sample_frames
|
||||
src_4h = VideoSourceMetadata(
|
||||
total_frames_num=432000, original_fps=30.0, duration=14400.0
|
||||
)
|
||||
v_4h = backend_cls.compute_frames_index_to_sample(
|
||||
src_4h, VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||
)
|
||||
h_4h = hf_proc.sample_frames(
|
||||
VideoMetadata(fps=30.0, duration=14400.0, total_num_frames=432000),
|
||||
fps=1.0,
|
||||
max_frames=32,
|
||||
overflow_strategy="uniform",
|
||||
).tolist()
|
||||
assert v_4h == h_4h, f"1fps/4h mismatch: {v_4h[-2:]} vs {h_4h[-2:]}"
|
||||
|
||||
|
||||
def test_video_loader_duplicate_expansion():
|
||||
from transformers.models.embedding_gemma2.video_processing_embedding_gemma2 import (
|
||||
EmbeddingGemma2VideoProcessor,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.multimodal.video import (
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoSourceMetadata,
|
||||
VideoTargetMetadata,
|
||||
)
|
||||
|
||||
backend_cls = VIDEO_LOADER_REGISTRY.name2class["embedding_gemma2"]
|
||||
hf_proc = EmbeddingGemma2VideoProcessor()
|
||||
|
||||
# 4s clip @ 0.5fps = 2 frames total. Requested target fps = 1.0 -> 4 frames.
|
||||
# Source has fewer frames than requested sampled frames, requiring duplicates.
|
||||
src_lowfps = VideoSourceMetadata(total_frames_num=2, original_fps=0.5, duration=4.0)
|
||||
tgt = VideoTargetMetadata(num_frames=-1, fps=1.0, max_duration=300)
|
||||
vllm_indices = backend_cls.compute_frames_index_to_sample(src_lowfps, tgt)
|
||||
hf_meta = VideoMetadata(fps=0.5, duration=4.0, total_num_frames=2)
|
||||
hf_indices = hf_proc.sample_frames(
|
||||
hf_meta, fps=1.0, max_frames=32, overflow_strategy="uniform"
|
||||
).tolist()
|
||||
assert vllm_indices == hf_indices
|
||||
assert len(vllm_indices) == 4
|
||||
assert vllm_indices == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_video_media_io_embedding_gemma2():
|
||||
"""Verify VideoMediaIO with default num_frames=32 loads video bytes for
|
||||
|
||||
EmbeddingGemma2 without error.
|
||||
"""
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.multimodal.media import MediaConnector
|
||||
from vllm.multimodal.media.video import VideoMediaIO
|
||||
|
||||
video_path = "/projects/gemma4-vllm/videos/test_intel.mp4"
|
||||
if not os.path.exists(video_path):
|
||||
return
|
||||
|
||||
with open(video_path, "rb") as f:
|
||||
data = f.read()
|
||||
|
||||
# VideoMediaIO defaults to num_frames=32
|
||||
video_io = VideoMediaIO(image_io=MagicMock(), video_backend="embedding_gemma2")
|
||||
loaded = video_io.load_bytes(data)
|
||||
frames, meta = loaded.media
|
||||
assert len(frames) == 32
|
||||
assert meta.get("do_sample_frames") is False
|
||||
|
||||
# MediaConnector path
|
||||
connector = MediaConnector(allowed_local_media_path="/projects/gemma4-vllm/videos")
|
||||
fetched = connector.fetch_video(
|
||||
f"file://{video_path}",
|
||||
video_processor="EmbeddingGemma2VideoProcessor",
|
||||
)
|
||||
frames_conn, meta_conn = fetched.media
|
||||
assert len(frames_conn) == 32
|
||||
assert meta_conn.get("do_sample_frames") is False
|
||||
|
||||
|
||||
def test_config_capping_and_explicit_limits():
|
||||
"""Verify EmbeddingGemma2ModelConfig backend selection, default capping,
|
||||
and preservation of explicit limits.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.model_executor.models.config import (
|
||||
EmbeddingGemma2ModelConfig,
|
||||
Gemma4Config,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
# (a) When attention backend is None, default to TRITON_ATTN,
|
||||
# and default capping: max_model_len > 8192 without explicit limit capped to 8192
|
||||
cfg_default = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=262144,
|
||||
original_max_model_len=None,
|
||||
),
|
||||
)
|
||||
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default)
|
||||
|
||||
assert cfg_default.attention_config.backend == AttentionBackendEnum.TRITON_ATTN
|
||||
assert cfg_default.model_config.max_model_len == 8192
|
||||
|
||||
# (b) Default capping with original_max_model_len == -1
|
||||
cfg_default_minus1 = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=131072,
|
||||
original_max_model_len=-1,
|
||||
),
|
||||
)
|
||||
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_default_minus1)
|
||||
|
||||
assert cfg_default_minus1.model_config.max_model_len == 8192
|
||||
|
||||
# (c) Explicit limit 4096 preserved
|
||||
cfg_explicit_4k = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(backend=None),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=4096,
|
||||
original_max_model_len=4096,
|
||||
),
|
||||
)
|
||||
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_4k)
|
||||
|
||||
assert cfg_explicit_4k.model_config.max_model_len == 4096
|
||||
|
||||
# (d) Explicit limit 16384 preserved and explicit backend preserved
|
||||
cfg_explicit_16k = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(backend=AttentionBackendEnum.FLASH_ATTN),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=16384,
|
||||
original_max_model_len=16384,
|
||||
),
|
||||
)
|
||||
with patch.object(Gemma4Config, "verify_and_update_config"):
|
||||
EmbeddingGemma2ModelConfig.verify_and_update_config(cfg_explicit_16k)
|
||||
|
||||
assert cfg_explicit_16k.attention_config.backend == AttentionBackendEnum.FLASH_ATTN
|
||||
assert cfg_explicit_16k.model_config.max_model_len == 16384
|
||||
|
||||
|
||||
def test_sentence_transformer_tokenizer_config_max_seq_length(tmp_path):
|
||||
"""Verify get_sentence_transformer_tokenizer_config accepts config
|
||||
with only max_seq_length.
|
||||
"""
|
||||
import json
|
||||
|
||||
from vllm.transformers_utils.config import (
|
||||
get_sentence_transformer_tokenizer_config,
|
||||
)
|
||||
|
||||
# 1. Synthetic model directory with only max_seq_length in sentence_bert_config.json
|
||||
cfg_file = tmp_path / "sentence_bert_config.json"
|
||||
cfg_file.write_text(json.dumps({"max_seq_length": 8192}))
|
||||
loaded = get_sentence_transformer_tokenizer_config(str(tmp_path))
|
||||
assert loaded is not None
|
||||
assert loaded["max_seq_length"] == 8192
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoConfig, AutoModel, CLIPImageProcessor, PreTrainedModel
|
||||
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
|
||||
# we use snapshot_download to prevent conflicts between
|
||||
# dynamic_module and trust_remote_code for hf_runner
|
||||
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_intern_vit_test(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
*,
|
||||
dtype: str,
|
||||
):
|
||||
model = hf_api().snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
|
||||
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
|
||||
|
||||
img_processor = CLIPImageProcessor.from_pretrained(model)
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
pixel_values = [
|
||||
img_processor(images, return_tensors="pt").pixel_values.to(torch_dtype)
|
||||
for images in images
|
||||
]
|
||||
|
||||
config = AutoConfig.from_pretrained(model, trust_remote_code=True)
|
||||
if not getattr(config, "norm_type", None):
|
||||
config.norm_type = "rms_norm"
|
||||
|
||||
# Monkey-patch for transformers v5 compatibility: InternVisionModel's
|
||||
# custom code doesn't call post_init(), so all_tied_weights_keys is
|
||||
# never set. Remove once https://github.com/OpenGVLab/InternVL fixes it.
|
||||
_orig_init = PreTrainedModel.__init__
|
||||
|
||||
def _patched_init(self, *args, **kwargs):
|
||||
_orig_init(self, *args, **kwargs)
|
||||
self.all_tied_weights_keys = {}
|
||||
|
||||
PreTrainedModel.__init__ = _patched_init
|
||||
|
||||
try:
|
||||
hf_model = AutoModel.from_pretrained(
|
||||
model, dtype=torch_dtype, trust_remote_code=True
|
||||
).to(DEVICE_TYPE)
|
||||
finally:
|
||||
PreTrainedModel.__init__ = _orig_init
|
||||
hf_outputs_per_image = [
|
||||
hf_model(pixel_value.to(DEVICE_TYPE)).last_hidden_state
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
|
||||
from vllm.model_executor.models.intern_vit import InternVisionModel
|
||||
|
||||
vllm_model = InternVisionModel(config)
|
||||
vllm_model.load_weights(hf_model.state_dict().items())
|
||||
|
||||
del hf_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
|
||||
vllm_outputs_per_image = [
|
||||
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
del vllm_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
cos_similar = nn.CosineSimilarity(dim=-1)
|
||||
for vllm_output, hf_output in zip(vllm_outputs_per_image, hf_outputs_per_image):
|
||||
assert cos_similar(vllm_output, hf_output).mean() > 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"OpenGVLab/InternViT-300M-448px",
|
||||
"OpenGVLab/InternViT-6B-448px-V1-5",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models(
|
||||
default_vllm_config, dist_init, image_assets, model_id, dtype: str
|
||||
) -> None:
|
||||
run_intern_vit_test(
|
||||
image_assets,
|
||||
model_id,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,332 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.assets.base import VLLM_S3_BUCKET_URL
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
|
||||
MODELS = ["jinaai/jina-reranker-m0"]
|
||||
|
||||
MM_PROCESSOR_KWARGS = {
|
||||
"min_pixels": 3136,
|
||||
"max_pixels": 602112,
|
||||
}
|
||||
|
||||
LIMIT_MM_PER_PROMPT = {"image": 2}
|
||||
|
||||
CHECKPOINT_TO_HF_MAPPER = {
|
||||
r"^visual\.": "model.visual.",
|
||||
r"^model\.(?!language_model\.|visual\.)": "model.language_model.",
|
||||
}
|
||||
|
||||
HANDELSBLATT_IMAGE_URL = (
|
||||
f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-handelsblatt-preview.png"
|
||||
)
|
||||
PAPER_IMAGE_URL = f"{VLLM_S3_BUCKET_URL}/multimodal_asset/jinavl-paper-11.png"
|
||||
|
||||
# Shared long text for test data
|
||||
LONG_TEXT_DOC = """We present ReaderLM-v2, a compact 1.5 billion parameter language model designed for efficient
|
||||
web content extraction. Our model processes documents up to 512K tokens, transforming messy HTML
|
||||
into clean Markdown or JSON formats with high accuracy -- making it an ideal tool for grounding
|
||||
large language models. The models effectiveness results from two key innovations: (1) a three-stage
|
||||
data synthesis pipeline that generates high quality, diverse training data by iteratively drafting,
|
||||
refining, and critiquing web content extraction; and (2) a unified training framework combining
|
||||
continuous pre-training with multi-objective optimization. Intensive evaluation demonstrates that
|
||||
ReaderLM-v2 outperforms GPT-4o-2024-08-06 and other larger models by 15-20% on carefully curated
|
||||
benchmarks, particularly excelling at documents exceeding 100K tokens, while maintaining significantly
|
||||
lower computational requirements.""" # noqa: E501
|
||||
|
||||
# Test data for different scenarios
|
||||
TEXT_IMAGE_TEST_DATA = {
|
||||
"query": [{"text": "slm markdown"}],
|
||||
"documents": [
|
||||
{"image": HANDELSBLATT_IMAGE_URL},
|
||||
{"image": PAPER_IMAGE_URL},
|
||||
],
|
||||
}
|
||||
|
||||
TEXT_TEXT_TEST_DATA = {
|
||||
"query": [{"text": "slm markdown"}],
|
||||
"documents": [
|
||||
{"text": LONG_TEXT_DOC},
|
||||
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
|
||||
],
|
||||
}
|
||||
|
||||
IMAGE_TEXT_TEST_DATA = {
|
||||
"query": [{"image": PAPER_IMAGE_URL}],
|
||||
"documents": [
|
||||
{"text": LONG_TEXT_DOC},
|
||||
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
|
||||
],
|
||||
}
|
||||
|
||||
IMAGE_IMAGE_TEST_DATA = {
|
||||
"query": [{"image": PAPER_IMAGE_URL}],
|
||||
"documents": [
|
||||
{"image": HANDELSBLATT_IMAGE_URL},
|
||||
{"image": PAPER_IMAGE_URL},
|
||||
],
|
||||
}
|
||||
|
||||
TEXT_MIXED_DOCS_TEST_DATA = {
|
||||
"query": [{"text": "slm markdown"}],
|
||||
"documents": [
|
||||
{"text": LONG_TEXT_DOC},
|
||||
{"image": PAPER_IMAGE_URL},
|
||||
{"text": "数据提取么?为什么不用正则啊,你用正则不就全解决了么?"},
|
||||
{"image": HANDELSBLATT_IMAGE_URL},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _normalize_image(image_val: str) -> str:
|
||||
"""Normalize image value to proper format for HF model."""
|
||||
return (
|
||||
image_val
|
||||
if image_val.startswith(("http://", "https://"))
|
||||
else f"data:image/png;base64,{image_val}"
|
||||
)
|
||||
|
||||
|
||||
def create_score_multimodal_param(
|
||||
content_parts: list[dict],
|
||||
) -> list[ScoreMultiModalParam]:
|
||||
"""Create a ScoreMultiModalParam from a list of content dictionaries.
|
||||
|
||||
Each dict supports the following formats:
|
||||
- Text: {'text': 'content'}
|
||||
- Image URL: {'image': 'https://...'}
|
||||
- Image Base64: {'image': 'base64_str'}
|
||||
"""
|
||||
formatted_content = []
|
||||
|
||||
for part in content_parts:
|
||||
if "text" in part:
|
||||
formatted_content.append(
|
||||
ChatCompletionContentPartTextParam(
|
||||
type="text",
|
||||
text=part["text"],
|
||||
)
|
||||
)
|
||||
elif "image" in part:
|
||||
image_val = part["image"]
|
||||
if image_val.startswith(("http://", "https://")):
|
||||
formatted_content.append(
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url={"url": image_val},
|
||||
)
|
||||
)
|
||||
else:
|
||||
formatted_content.append(
|
||||
ChatCompletionContentPartImageEmbedsParam(
|
||||
type="image_embeds", image_embeds=image_val
|
||||
)
|
||||
)
|
||||
|
||||
return [ScoreMultiModalParam(content=[content]) for content in formatted_content]
|
||||
|
||||
|
||||
def _run_vllm(
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
query_strs: list[dict[str, str]],
|
||||
document_strs: list[dict[str, str]],
|
||||
) -> list[float]:
|
||||
"""Run vLLM reranker and return scores."""
|
||||
query = create_score_multimodal_param(query_strs)
|
||||
documents = create_score_multimodal_param(document_strs)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
max_num_seqs=2,
|
||||
max_model_len=2048,
|
||||
mm_processor_kwargs=MM_PROCESSOR_KWARGS,
|
||||
limit_mm_per_prompt=LIMIT_MM_PER_PROMPT,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.llm.score(query, documents)
|
||||
|
||||
return [output.outputs.score for output in outputs]
|
||||
|
||||
|
||||
def _run_hf(
|
||||
hf_runner: type[HfRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
query_strs: list[dict[str, str]],
|
||||
document_strs: list[dict[str, str]],
|
||||
) -> list[float]:
|
||||
"""Run HuggingFace reranker and return scores."""
|
||||
query = query_strs[0]
|
||||
if "text" in query:
|
||||
query_type = "text"
|
||||
query_data = query["text"]
|
||||
elif "image" in query:
|
||||
query_type = "image"
|
||||
query_data = _normalize_image(query["image"])
|
||||
else:
|
||||
raise ValueError("Unsupported query format")
|
||||
|
||||
scores: list[float] = []
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
trust_remote_code=True,
|
||||
auto_cls=AutoModel,
|
||||
model_kwargs={"key_mapping": CHECKPOINT_TO_HF_MAPPER},
|
||||
) as hf_model:
|
||||
for doc in document_strs:
|
||||
if "text" in doc:
|
||||
score = hf_model.model.compute_score(
|
||||
[[query_data, doc["text"]]],
|
||||
max_length=2048,
|
||||
query_type=query_type,
|
||||
doc_type="text",
|
||||
)
|
||||
scores.append(score)
|
||||
elif "image" in doc:
|
||||
score = hf_model.model.compute_score(
|
||||
[[query_data, doc["image"]]],
|
||||
max_length=2048,
|
||||
query_type=query_type,
|
||||
doc_type="image",
|
||||
)
|
||||
scores.append(score)
|
||||
return scores
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
query_strs: list[dict[str, str]],
|
||||
document_strs: list[dict[str, str]],
|
||||
) -> None:
|
||||
"""Run comparison test between vLLM and HuggingFace implementations."""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
vllm_outputs = _run_vllm(vllm_runner, model, dtype, query_strs, document_strs)
|
||||
hf_outputs = _run_hf(hf_runner, model, dtype, query_strs, document_strs)
|
||||
|
||||
# Compare outputs
|
||||
assert len(hf_outputs) == len(vllm_outputs), (
|
||||
f"Output length mismatch: HF={len(hf_outputs)}, vLLM={len(vllm_outputs)}"
|
||||
)
|
||||
|
||||
for i, (hf_score, vllm_score) in enumerate(zip(hf_outputs, vllm_outputs)):
|
||||
assert hf_score == pytest.approx(vllm_score, rel=0.02), (
|
||||
f"Score mismatch at index {i}: HF={hf_score}, vLLM={vllm_score}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_model_text_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Visual Documents Reranking."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype,
|
||||
TEXT_IMAGE_TEST_DATA["query"],
|
||||
TEXT_IMAGE_TEST_DATA["documents"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_model_text_text(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Textual Documents Reranking."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype,
|
||||
TEXT_TEXT_TEST_DATA["query"],
|
||||
TEXT_TEXT_TEST_DATA["documents"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_model_image_text(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Image Querying for Textual Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype,
|
||||
IMAGE_TEXT_TEST_DATA["query"],
|
||||
IMAGE_TEXT_TEST_DATA["documents"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_model_image_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Image Querying for Image Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype,
|
||||
IMAGE_IMAGE_TEST_DATA["query"],
|
||||
IMAGE_IMAGE_TEST_DATA["documents"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_model_text_mixed_documents(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Text Query for Mixed Text and Image Documents."""
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype,
|
||||
TEXT_MIXED_DOCS_TEST_DATA["query"],
|
||||
TEXT_MIXED_DOCS_TEST_DATA["documents"],
|
||||
)
|
||||
@@ -1,394 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the LlamaNemotronVL model family:
|
||||
- nvidia/llama-nemotron-embed-vl-1b-v2 (LlamaNemotronVLForCausalLM / embed)
|
||||
- nvidia/llama-nemotron-rerank-vl-1b-v2
|
||||
(LlamaNemotronVLForSequenceClassification / rerank)
|
||||
|
||||
Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoModel, AutoModelForSequenceClassification, AutoProcessor
|
||||
|
||||
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
from vllm.transformers_utils.processors.nemotron_vl import (
|
||||
SIGLIP_MEAN,
|
||||
SIGLIP_STD,
|
||||
LlamaNemotronVLEmbedImageProcessor,
|
||||
)
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ....utils import ROCM_ENGINE_KWARGS
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
# Prefixes used by the model API
|
||||
QUERY_PREFIX = "query: "
|
||||
PASSAGE_PREFIX = "passage: "
|
||||
|
||||
# Text prompts for text-only embedding
|
||||
HF_TEXT_PROMPTS = [
|
||||
# T -> X (text embedding queries)
|
||||
f"{QUERY_PREFIX}The label of the object is stop sign",
|
||||
f"{QUERY_PREFIX}cherry blossom",
|
||||
]
|
||||
|
||||
# Image prompts using the model's expected format
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
# I -> X (image embedding as passage/document)
|
||||
"stop_sign": f"{PASSAGE_PREFIX}<image>",
|
||||
"cherry_blossom": f"{PASSAGE_PREFIX}<image>",
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = ["nvidia/llama-nemotron-embed-vl-1b-v2"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_device_normalization_matches_cpu_transform() -> None:
|
||||
device = current_platform.device_type
|
||||
image = Image.new("RGB", (48, 32), color=(12, 128, 240))
|
||||
processor_kwargs = dict(
|
||||
image_size=32,
|
||||
min_dynamic_patch=1,
|
||||
max_dynamic_patch=1,
|
||||
dynamic_image_size=True,
|
||||
use_thumbnail=False,
|
||||
)
|
||||
|
||||
cpu_processor = LlamaNemotronVLEmbedImageProcessor(**processor_kwargs)
|
||||
device_processor = LlamaNemotronVLEmbedImageProcessor(
|
||||
**processor_kwargs, do_rescale=False, do_normalize=False
|
||||
)
|
||||
|
||||
expected = cpu_processor._images_to_pixel_values_lst([image])[0]
|
||||
raw_pixels = device_processor._images_to_pixel_values_lst([image])[0]
|
||||
input_norm = FusedMMInputNorm(
|
||||
image_mean=list(SIGLIP_MEAN),
|
||||
image_std=list(SIGLIP_STD),
|
||||
rescale_factor=1 / 255,
|
||||
).to(device)
|
||||
actual = input_norm(
|
||||
raw_pixels.flatten(start_dim=1).to(device), expected.dtype
|
||||
).view_as(expected)
|
||||
|
||||
assert raw_pixels.dtype == torch.uint8
|
||||
torch.testing.assert_close(actual, expected.to(device))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("do_rescale", "do_normalize"), [(True, False), (False, True)])
|
||||
def test_device_normalization_rejects_partial_cpu_processing(
|
||||
do_rescale: bool, do_normalize: bool
|
||||
) -> None:
|
||||
with pytest.raises(ValueError, match="both enabled or both disabled"):
|
||||
LlamaNemotronVLEmbedImageProcessor(
|
||||
image_size=32,
|
||||
min_dynamic_patch=1,
|
||||
max_dynamic_patch=1,
|
||||
dynamic_image_size=True,
|
||||
use_thumbnail=False,
|
||||
do_rescale=do_rescale,
|
||||
do_normalize=do_normalize,
|
||||
)
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_cases: list[tuple[list[str], PromptImageInput]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Compare HF and vLLM embeddings for all input cases.
|
||||
|
||||
NOTE: Run vLLM first to avoid CUDA initialization issues with multiprocessing.
|
||||
"""
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
**ROCM_ENGINE_KWARGS,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.embed(input_texts, images=input_images)
|
||||
for input_texts, input_images in input_cases
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs_per_case = []
|
||||
for input_texts, input_images in input_cases:
|
||||
hf_outputs = []
|
||||
for text, image in zip(input_texts, input_images):
|
||||
with torch.inference_mode():
|
||||
if text.startswith(QUERY_PREFIX):
|
||||
query_text = text[len(QUERY_PREFIX) :]
|
||||
embedding = hf_model.model.encode_queries([query_text])
|
||||
elif text.startswith(PASSAGE_PREFIX):
|
||||
passage_text = text[len(PASSAGE_PREFIX) :]
|
||||
if image is not None:
|
||||
embedding = hf_model.model.encode_documents(
|
||||
images=[image],
|
||||
texts=[passage_text],
|
||||
)
|
||||
else:
|
||||
embedding = hf_model.model.encode_documents(
|
||||
texts=[passage_text]
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Text must start with {QUERY_PREFIX!r} "
|
||||
f"or {PASSAGE_PREFIX!r}"
|
||||
)
|
||||
|
||||
hf_outputs.append(embedding[0].tolist())
|
||||
hf_outputs_per_case.append(hf_outputs)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Test text and image embedding."""
|
||||
text_images = [None] * len(HF_TEXT_PROMPTS)
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
input_cases = [
|
||||
(HF_TEXT_PROMPTS, text_images),
|
||||
(HF_IMAGE_PROMPTS, images),
|
||||
]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reranker tests — nvidia/llama-nemotron-rerank-vl-1b-v2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RERANKER_MODELS = ["nvidia/llama-nemotron-rerank-vl-1b-v2"]
|
||||
|
||||
# The tokenizer's built-in chat template is not suitable for the Score/Rerank
|
||||
# APIs (it's inherited from the base LLM). We must use the provided override.
|
||||
_RERANKER_SCORE_TEMPLATE = (
|
||||
Path(__file__).parents[4]
|
||||
/ "examples/pooling/score/template/nemotron-vl-rerank.jinja"
|
||||
).read_text()
|
||||
|
||||
RERANKER_TEXT_QUERY = "How is AI improving the intelligence and capabilities of robots?"
|
||||
RERANKER_TEXT_DOCS = [
|
||||
"AI enables robots to perceive, plan, and act autonomously.",
|
||||
(
|
||||
"A biological foundation model designed to analyze DNA, RNA, "
|
||||
"and protein sequences."
|
||||
),
|
||||
]
|
||||
|
||||
RERANKER_IMAGE_QUERY = "photo of a red stop sign on a street"
|
||||
|
||||
RerankerDocument = tuple[str | None, Image.Image | None]
|
||||
RerankerCase = tuple[str, Sequence[RerankerDocument]]
|
||||
|
||||
|
||||
def _pil_to_data_uri(image: Image.Image) -> str:
|
||||
buf = BytesIO()
|
||||
image.save(buf, format="PNG")
|
||||
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||
return f"data:image/png;base64,{b64}"
|
||||
|
||||
|
||||
def _run_hf_reranker(
|
||||
hf_runner: type[HfRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
input_cases: Sequence[RerankerCase],
|
||||
) -> list[list[float]]:
|
||||
"""Run all HF reranker cases in one model lifecycle."""
|
||||
with hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
trust_remote_code=True,
|
||||
auto_cls=AutoModelForSequenceClassification,
|
||||
) as hf_model:
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
model,
|
||||
trust_remote_code=True,
|
||||
max_input_tiles=6,
|
||||
use_thumbnail=True,
|
||||
rerank_max_length=2048,
|
||||
)
|
||||
scores_per_case = []
|
||||
for query, docs in input_cases:
|
||||
examples = [
|
||||
{
|
||||
"question": query,
|
||||
"doc_text": doc_text if doc_text is not None else "",
|
||||
"doc_image": doc_image if doc_image is not None else "",
|
||||
}
|
||||
for doc_text, doc_image in docs
|
||||
]
|
||||
batch_dict = processor.process_queries_documents_crossencoder(examples)
|
||||
batch_dict = {
|
||||
k: v.to(hf_model.model.device) if isinstance(v, torch.Tensor) else v
|
||||
for k, v in batch_dict.items()
|
||||
}
|
||||
with torch.inference_mode():
|
||||
logits = hf_model.model(**batch_dict, return_dict=True).logits
|
||||
# vLLM applies sigmoid activation to raw logits before returning scores.
|
||||
scores = torch.sigmoid(logits.squeeze(-1).float())
|
||||
scores_per_case.append(scores.detach().cpu().tolist())
|
||||
|
||||
return scores_per_case
|
||||
|
||||
|
||||
def _run_vllm_reranker(
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
input_cases: Sequence[RerankerCase],
|
||||
) -> list[list[float]]:
|
||||
"""Run all vLLM reranker cases in one model lifecycle."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
**ROCM_ENGINE_KWARGS,
|
||||
) as vllm_model:
|
||||
scores_per_case = []
|
||||
for query, docs in input_cases:
|
||||
has_images = any(img is not None for _, img in docs)
|
||||
|
||||
if not has_images:
|
||||
queries = [query] * len(docs)
|
||||
doc_texts = [doc_text for doc_text, _ in docs]
|
||||
outputs = vllm_model.score(
|
||||
queries,
|
||||
doc_texts,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
else:
|
||||
query_params = [
|
||||
ScoreMultiModalParam(
|
||||
content=[
|
||||
ChatCompletionContentPartTextParam(
|
||||
type="text",
|
||||
text=query,
|
||||
)
|
||||
]
|
||||
)
|
||||
] * len(docs)
|
||||
|
||||
doc_params = []
|
||||
for doc_text, doc_image in docs:
|
||||
content: list = []
|
||||
if doc_image is not None:
|
||||
content.append(
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url={"url": _pil_to_data_uri(doc_image)},
|
||||
)
|
||||
)
|
||||
if doc_text:
|
||||
content.append(
|
||||
ChatCompletionContentPartTextParam(
|
||||
type="text",
|
||||
text=doc_text,
|
||||
)
|
||||
)
|
||||
doc_params.append(ScoreMultiModalParam(content=content))
|
||||
|
||||
raw_outputs = vllm_model.llm.score(
|
||||
query_params,
|
||||
doc_params,
|
||||
chat_template=_RERANKER_SCORE_TEMPLATE,
|
||||
)
|
||||
outputs = [output.outputs.score for output in raw_outputs]
|
||||
|
||||
scores_per_case.append(outputs)
|
||||
|
||||
return scores_per_case
|
||||
|
||||
|
||||
def _run_reranker_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
dtype: str,
|
||||
input_cases: Sequence[RerankerCase],
|
||||
) -> None:
|
||||
"""Compare HF and vLLM reranker scores for all input cases.
|
||||
|
||||
NOTE: Run vLLM first to avoid CUDA initialization issues with multiprocessing.
|
||||
"""
|
||||
vllm_scores_per_case = _run_vllm_reranker(vllm_runner, model, dtype, input_cases)
|
||||
hf_scores_per_case = _run_hf_reranker(hf_runner, model, dtype, input_cases)
|
||||
|
||||
# ROCm has slightly higher variance because vLLM and HF use different
|
||||
# attention backends.
|
||||
rel_tol = 0.022 if current_platform.is_rocm() else 0.02
|
||||
for hf_scores, vllm_scores in zip(hf_scores_per_case, vllm_scores_per_case):
|
||||
assert len(hf_scores) == len(vllm_scores), (
|
||||
f"Output length mismatch: HF={len(hf_scores)}, vLLM={len(vllm_scores)}"
|
||||
)
|
||||
for i, (hf_score, vllm_score) in enumerate(zip(hf_scores, vllm_scores)):
|
||||
assert hf_score == pytest.approx(vllm_score, rel=rel_tol), (
|
||||
f"Score mismatch at index {i}: HF={hf_score:.4f}, vLLM={vllm_score:.4f}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", RERANKER_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_reranker(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Test reranking with text and image documents."""
|
||||
text_docs: list[RerankerDocument] = [(text, None) for text in RERANKER_TEXT_DOCS]
|
||||
image_docs: list[RerankerDocument] = [
|
||||
(None, asset.pil_image) for asset in image_assets
|
||||
]
|
||||
input_cases: list[RerankerCase] = [
|
||||
(RERANKER_TEXT_QUERY, text_docs),
|
||||
(RERANKER_IMAGE_QUERY, image_docs),
|
||||
]
|
||||
_run_reranker_test(hf_runner, vllm_runner, model, dtype, input_cases)
|
||||
@@ -1,159 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch.nn.functional as F
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
# Llava Next embedding implementation is only supported by CUDA.
|
||||
# If run on ROCm, hf_model.model.resize_token_embeddings will
|
||||
# cause the following error:
|
||||
# RuntimeError: Calling torch.linalg.cholesky on a CUDA tensor
|
||||
# requires compiling PyTorch with MAGMA. Please use PyTorch
|
||||
# built with MAGMA support.
|
||||
# If run on CPU, hf_model.model.resize_token_embeddings will
|
||||
# cause the following error:
|
||||
# RuntimeError: Calling torch.linalg.cholesky on a CPU tensor
|
||||
# requires compiling PyTorch with LAPACK. Please use PyTorch
|
||||
# built with LAPACK support.
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Llava Next model uses op that is only supported in CUDA",
|
||||
)
|
||||
|
||||
llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n \n" # noqa: E501
|
||||
|
||||
HF_TEXT_PROMPTS = [
|
||||
# T -> X
|
||||
llama3_template.format(
|
||||
"The label of the object is stop sign\nSummary above sentence in one word: " # noqa: E501
|
||||
),
|
||||
# T -> X
|
||||
llama3_template.format("cherry blossom\nSummary above sentence in one word: "),
|
||||
]
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
# I -> X
|
||||
"stop_sign": llama3_template.format(
|
||||
"<image>\nSummary above image in one word: "
|
||||
),
|
||||
# I -> X
|
||||
"cherry_blossom": llama3_template.format(
|
||||
"<image>\nSummary above image in one word: "
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = ["royokong/e5-v"]
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_texts: list[str],
|
||||
input_images: PromptImageInput,
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
with vllm_runner(
|
||||
model, runner="pooling", dtype=dtype, max_model_len=4096, enforce_eager=True
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(input_texts, images=input_images)
|
||||
|
||||
with hf_runner(
|
||||
model, dtype=dtype, auto_cls=AutoModelForImageTextToText
|
||||
) as hf_model:
|
||||
# Patch the issue where generation_config.json is missing
|
||||
hf_model.processor.patch_size = hf_model.model.config.vision_config.patch_size
|
||||
|
||||
# Patch the issue where image_token_id
|
||||
# exceeds the maximum allowed vocab size
|
||||
hf_model.model.resize_token_embeddings(
|
||||
hf_model.model.model.language_model.vocab_size + 1
|
||||
)
|
||||
|
||||
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
|
||||
|
||||
all_outputs = []
|
||||
for inputs in all_inputs:
|
||||
# Based on: https://huggingface.co/royokong/e5-v
|
||||
outputs = hf_model.model(
|
||||
**hf_model.wrap_device(inputs),
|
||||
return_dict=True,
|
||||
output_hidden_states=True,
|
||||
)
|
||||
pooled_output = F.normalize(outputs.hidden_states[-1][0, -1, :], dim=-1)
|
||||
|
||||
all_outputs.append(pooled_output.tolist())
|
||||
|
||||
hf_outputs = all_outputs
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models_text(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images, # type: ignore
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=48)
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [
|
||||
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
|
||||
]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import transformers.utils
|
||||
from PIL import Image
|
||||
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.assets.image import VLM_IMAGES_DIR
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ....utils import large_gpu_test
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
# BC for method that was deleted in Transformers v5.
|
||||
# Only needed for generating the HF reference.
|
||||
transformers.utils.is_flash_attn_greater_or_equal_2_10 = (
|
||||
lambda: transformers.utils.is_flash_attn_greater_or_equal("2.1.0")
|
||||
)
|
||||
|
||||
HF_TEXT_PROMPTS = [
|
||||
# T -> X
|
||||
"Find me an everyday image that matches the given caption: The label of the object is stop sign", # noqa: E501
|
||||
# T -> X
|
||||
"Retrieve an image of this caption: cherry blossom",
|
||||
]
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
# T + I -> X
|
||||
"stop_sign": "<|image_1|> Select the portion of the image that isolates the object of the given label: The label of the object is stop sign", # noqa: E501
|
||||
# I -> X
|
||||
"cherry_blossom": "<|image_1|> Represent the given image for classification", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = ["TIGER-Lab/VLM2Vec-Full"]
|
||||
|
||||
SPECIAL_TOKEN_IMAGE_PROMPT = (
|
||||
"\n<s><|user|>\n <|image_1|>\n\t <s>"
|
||||
"Represent the given image for classification<|end|>"
|
||||
"\n<|assistant|>\n"
|
||||
)
|
||||
|
||||
|
||||
def _get_cherry_blossom_image() -> Image.Image:
|
||||
return Image.open(
|
||||
get_vllm_public_assets(filename="cherry_blossom.jpg", s3_prefix=VLM_IMAGES_DIR)
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_texts: list[str],
|
||||
input_images: PromptImageInput,
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
with vllm_runner(
|
||||
model, runner="pooling", dtype=dtype, enforce_eager=True
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(input_texts, images=input_images)
|
||||
|
||||
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
|
||||
hf_model_kwargs = {"_attn_implementation": "eager"}
|
||||
with hf_runner(model, dtype=dtype, model_kwargs=hf_model_kwargs) as hf_model:
|
||||
all_inputs = hf_model.get_inputs(input_texts, images=input_images)
|
||||
|
||||
all_outputs = []
|
||||
for inputs in all_inputs:
|
||||
# Based on: https://github.com/TIGER-AI-Lab/VLM2Vec/blob/db3b951bccabba220c1f53ab46a734e50dd2fc08/src/model.py
|
||||
outputs = hf_model.model(
|
||||
**hf_model.wrap_device(inputs),
|
||||
return_dict=True,
|
||||
output_hidden_states=True,
|
||||
)
|
||||
last_hidden_state = outputs.hidden_states[-1][0]
|
||||
reps = last_hidden_state[inputs.attention_mask[0].sum() - 1]
|
||||
pooled_output = F.normalize(reps, p=2, dim=-1)
|
||||
|
||||
all_outputs.append(pooled_output.tolist())
|
||||
|
||||
hf_outputs = all_outputs
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_xpu(),
|
||||
reason=(
|
||||
"vllm-xpu-kernels paged_decode_default.conf is missing the "
|
||||
"8,96,64,false,true,false tuple needed by VLM2Vec-Full at "
|
||||
"block_size=64. Re-enable once vllm-xpu-kernels ships the updated "
|
||||
"config. See github.com/vllm-project/vllm-xpu-kernels/pull/587."
|
||||
),
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models_text(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [(text, None) for text in HF_TEXT_PROMPTS]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images, # type: ignore
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
@large_gpu_test(min_gb=48)
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
input_texts_images = [
|
||||
(text, asset.pil_image) for text, asset in zip(HF_IMAGE_PROMPTS, image_assets)
|
||||
]
|
||||
input_texts = [text for text, _ in input_texts_images]
|
||||
input_images = [image for _, image in input_texts_images]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_texts,
|
||||
input_images,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
def test_models_image_special_tokens_processing(
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
model_config = ModelConfig(
|
||||
model,
|
||||
runner="pooling",
|
||||
trust_remote_code=True,
|
||||
dtype=dtype,
|
||||
max_model_len=1024,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
image = _get_cherry_blossom_image()
|
||||
|
||||
processed_inputs = processor(
|
||||
SPECIAL_TOKEN_IMAGE_PROMPT,
|
||||
mm_items=processor.info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
hf_inputs = hf_processor(
|
||||
SPECIAL_TOKEN_IMAGE_PROMPT,
|
||||
images=image,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
image_token_id = hf_processor.get_special_image_token_id()
|
||||
hf_prompt_token_ids = [
|
||||
image_token_id if token_id < 0 else token_id
|
||||
for token_id in hf_inputs["input_ids"][0].tolist()
|
||||
]
|
||||
|
||||
prompt_token_ids = processed_inputs["prompt_token_ids"]
|
||||
|
||||
assert prompt_token_ids == hf_prompt_token_ids
|
||||
assert prompt_token_ids.count(image_token_id) == hf_prompt_token_ids.count(
|
||||
image_token_id
|
||||
)
|
||||
assert prompt_token_ids.count(image_token_id) > 0
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from ....conftest import VllmRunner
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
importlib.util.find_spec("terratorch") is None,
|
||||
reason="terratorch unavailable while PyPI has `lightning` quarantined; see #41376",
|
||||
)
|
||||
|
||||
|
||||
def _run_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
model: str,
|
||||
) -> None:
|
||||
prompt = [
|
||||
{
|
||||
# This model deals with no text input
|
||||
"prompt_token_ids": [1],
|
||||
"multi_modal_data": {
|
||||
"image": {
|
||||
"pixel_values": torch.ones((6, 512, 512), dtype=torch.float16),
|
||||
"location_coords": torch.ones((1, 2), dtype=torch.float16),
|
||||
}
|
||||
},
|
||||
}
|
||||
for _ in range(10)
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="half",
|
||||
enforce_eager=True,
|
||||
skip_tokenizer_init=True,
|
||||
enable_mm_embeds=True,
|
||||
# Limit the maximum number of sequences to avoid the
|
||||
# test going OOM during the warmup run
|
||||
max_num_seqs=32,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
vllm_model.llm.encode(prompt, pooling_task="plugin")
|
||||
|
||||
|
||||
MODELS = ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_models_image(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
) -> None:
|
||||
_run_test(
|
||||
vllm_runner,
|
||||
model,
|
||||
)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
MODEL = "Qwen/Qwen3-ForcedAligner-0.6B"
|
||||
CLASSIFY_NUM = 5000
|
||||
TIMESTAMP_TOKEN_ID = 151705
|
||||
|
||||
|
||||
def build_prompt(words: list[str]) -> str:
|
||||
body = "<timestamp><timestamp>".join(words) + "<timestamp><timestamp>"
|
||||
return f"<|audio_start|><|audio_pad|><|audio_end|>{body}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [MODEL])
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@torch.inference_mode()
|
||||
def test_qwen3_forced_aligner(
|
||||
vllm_runner,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
words = ["Hello", "world"]
|
||||
prompt = build_prompt(words)
|
||||
|
||||
# 5-second silent audio at 16kHz
|
||||
audio = np.zeros(16000 * 5, dtype=np.float32)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
max_model_len=512,
|
||||
hf_overrides={
|
||||
"architectures": [
|
||||
"Qwen3ASRForcedAlignerForTokenClassification",
|
||||
],
|
||||
},
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.llm.encode(
|
||||
[{"prompt": prompt, "multi_modal_data": {"audio": audio}}],
|
||||
pooling_task="token_classify",
|
||||
)
|
||||
|
||||
# Validate output structure
|
||||
assert len(outputs) == 1
|
||||
logits = outputs[0].outputs.data
|
||||
assert logits.dim() == 2
|
||||
assert logits.shape[1] == CLASSIFY_NUM
|
||||
|
||||
# Validate timestamp extraction
|
||||
token_ids = outputs[0].prompt_token_ids
|
||||
predictions = logits.argmax(dim=-1)
|
||||
ts_indices = [i for i, t in enumerate(token_ids) if t == TIMESTAMP_TOKEN_ID]
|
||||
|
||||
# 2 words x 2 timestamps each (start + end) = 4
|
||||
assert len(ts_indices) == 4
|
||||
|
||||
ts_preds = [predictions[i].item() for i in ts_indices]
|
||||
assert all(p >= 0 for p in ts_preds)
|
||||
# end >= start for each word
|
||||
assert ts_preds[1] >= ts_preds[0] # Hello
|
||||
assert ts_preds[3] >= ts_preds[2] # world
|
||||
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
|
||||
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.models.radio import RadioModel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.configs.radio import RadioConfig
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
|
||||
# we use snapshot_download to prevent conflicts between
|
||||
# dynamic_module and trust_remote_code for hf_runner
|
||||
DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def run_radio_test(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
*,
|
||||
dtype: str,
|
||||
):
|
||||
model = hf_api().snapshot_download(model_id, allow_patterns=DOWNLOAD_PATTERN)
|
||||
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
|
||||
|
||||
img_processor = CLIPImageProcessor.from_pretrained(model)
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
# Input resolution must be a multiple of `self.min_resolution_step`.
|
||||
# Using `self.get_nearest_supported_resolution`, for assets 432x642 the
|
||||
# nearest supported resolution is 432x640.
|
||||
pixel_values = [
|
||||
img_processor(image, return_tensors="pt").pixel_values.to(torch_dtype)[
|
||||
:, :, :, :640
|
||||
]
|
||||
for image in images
|
||||
]
|
||||
|
||||
hf_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
|
||||
|
||||
# RADIO model on HF does not properly handle torch_dtype argument
|
||||
# And relies on args["dtype"] which we have to patch manually:
|
||||
hf_config.args["dtype"] = torch_dtype
|
||||
|
||||
hf_model = AutoModel.from_pretrained(
|
||||
model_id,
|
||||
config=hf_config,
|
||||
dtype=torch_dtype,
|
||||
trust_remote_code=True,
|
||||
).to(DEVICE_TYPE)
|
||||
hf_model.eval()
|
||||
|
||||
# A HF model has image normalization as a part of model's forward
|
||||
# However in vLLM we don't make normalization a part of the model
|
||||
# forward step since mean/std stored as model's parameters and
|
||||
# subject to precision loss (when using fp16/bf16) which negatively
|
||||
# affects evaluation benchmarks.
|
||||
hf_model.make_preprocessor_external()
|
||||
|
||||
hf_outputs_per_image = [
|
||||
hf_model(pixel_value.to(DEVICE_TYPE)) for pixel_value in pixel_values
|
||||
]
|
||||
|
||||
vllm_config = RadioConfig(
|
||||
model_name=hf_config.args["model"],
|
||||
**hf_config.args,
|
||||
)
|
||||
vllm_model = RadioModel(vllm_config)
|
||||
vllm_model.load_weights(hf_model.state_dict())
|
||||
vllm_model = vllm_model.to(DEVICE_TYPE, torch_dtype)
|
||||
|
||||
vllm_outputs_per_image = [
|
||||
vllm_model(pixel_values=pixel_value.to(DEVICE_TYPE))
|
||||
for pixel_value in pixel_values
|
||||
]
|
||||
del vllm_model, hf_model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
cos_similar = nn.CosineSimilarity(dim=-1)
|
||||
for vllm_output, hf_output in zip(vllm_outputs_per_image, hf_outputs_per_image):
|
||||
assert cos_similar(vllm_output[0], hf_output[0]).mean() > 0.99
|
||||
assert cos_similar(vllm_output[1], hf_output[1]).mean() > 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"nvidia/C-RADIOv2-H",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
|
||||
def test_radio(
|
||||
default_vllm_config, dist_init, image_assets, model_id, dtype: str
|
||||
) -> None:
|
||||
run_radio_test(
|
||||
image_assets,
|
||||
model_id,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import SiglipModel
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
|
||||
from ...utils import check_embeddings_close
|
||||
|
||||
HF_TEXT_PROMPTS = [
|
||||
"a photo of a stop sign",
|
||||
"a photo of a cherry blossom",
|
||||
]
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "",
|
||||
"cherry_blossom": "",
|
||||
}
|
||||
)
|
||||
|
||||
MODELS = [
|
||||
"google/siglip-base-patch16-224",
|
||||
"google/siglip2-base-patch16-224",
|
||||
# Different image embedding dim than text_config.hidden_size
|
||||
"google/siglip2-giant-opt-patch16-384",
|
||||
]
|
||||
|
||||
|
||||
def _run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
input_cases: list[tuple[list[str], PromptImageInput, dict[str, Any]]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype=dtype,
|
||||
enforce_eager=True,
|
||||
max_model_len=64,
|
||||
gpu_memory_utilization=0.7,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.embed(
|
||||
input_texts,
|
||||
images=input_images,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
for input_texts, input_images, tokenization_kwargs in input_cases
|
||||
]
|
||||
|
||||
texts = [HF_TEXT_PROMPTS[0]]
|
||||
images = [input_cases[1][1][0]]
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
vllm_model.embed(texts, images=images)
|
||||
|
||||
vllm_model.embed(texts)
|
||||
vllm_model.embed([""], images=images)
|
||||
|
||||
mixed_outputs = vllm_outputs_per_case[2]
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[0][0]],
|
||||
embeddings_1_lst=[mixed_outputs[0]],
|
||||
name_0="text_only",
|
||||
name_1="mixed_text",
|
||||
)
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=[vllm_outputs_per_case[1][0]],
|
||||
embeddings_1_lst=[mixed_outputs[1]],
|
||||
name_0="image_only",
|
||||
name_1="mixed_image",
|
||||
)
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=SiglipModel) as hf_model:
|
||||
hf_outputs_per_case = []
|
||||
for input_texts, input_images, tokenization_kwargs in input_cases:
|
||||
all_inputs = hf_model.get_inputs(
|
||||
input_texts,
|
||||
images=input_images,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
hf_outputs = []
|
||||
for inputs in all_inputs:
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
|
||||
if "pixel_values" in inputs:
|
||||
pooled_output = hf_model.model.get_image_features(
|
||||
pixel_values=inputs.pixel_values,
|
||||
)
|
||||
else:
|
||||
pooled_output = hf_model.model.get_text_features(
|
||||
input_ids=inputs.input_ids,
|
||||
)
|
||||
|
||||
if not isinstance(pooled_output, torch.Tensor):
|
||||
pooled_output = pooled_output.pooler_output
|
||||
pooled_output = pooled_output.squeeze(0)
|
||||
hf_outputs.append(pooled_output.tolist())
|
||||
|
||||
hf_outputs_per_case.append(hf_outputs)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
model: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
text_images = [None] * len(HF_TEXT_PROMPTS)
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
input_cases = [
|
||||
(
|
||||
HF_TEXT_PROMPTS,
|
||||
text_images,
|
||||
{
|
||||
"padding": "max_length",
|
||||
"max_length": 64,
|
||||
},
|
||||
),
|
||||
(HF_IMAGE_PROMPTS, images, {}),
|
||||
(
|
||||
[HF_TEXT_PROMPTS[0], HF_IMAGE_PROMPTS[0]],
|
||||
[None, images[0]],
|
||||
{
|
||||
"padding": "max_length",
|
||||
"max_length": 64,
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
_run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
input_cases,
|
||||
model,
|
||||
dtype=dtype,
|
||||
)
|
||||
@@ -1,154 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2025 The vLLM team.
|
||||
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
|
||||
# reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
|
||||
|
||||
class MockAudioFlamingo3Config(PreTrainedConfig):
|
||||
model_type = "audioflamingo3"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.audio_config = PreTrainedConfig()
|
||||
self.text_config = PreTrainedConfig()
|
||||
|
||||
|
||||
class MockAudioFlamingo3Processor:
|
||||
def __init__(self):
|
||||
self.audio_token = "<sound>"
|
||||
self.audio_token_id = 12345
|
||||
self.max_audio_len = 60
|
||||
self.feature_extractor = MockFeatureExtractor()
|
||||
self.tokenizer = self._tokenize
|
||||
|
||||
def __call__(self, text=None, audio=None, **kwargs):
|
||||
return {
|
||||
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
|
||||
"input_features": torch.zeros((3, 80, 3000)),
|
||||
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
def _tokenize(self, text, **kwargs):
|
||||
return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
|
||||
|
||||
|
||||
class MockFeatureExtractor:
|
||||
def __init__(self):
|
||||
self.sampling_rate = 16000
|
||||
self.chunk_length = 30
|
||||
self.hop_length = 160
|
||||
|
||||
def __call__(self, audios, **kwargs):
|
||||
return {
|
||||
"input_features": torch.zeros((len(audios), 80, 3000)),
|
||||
"attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
config = MockAudioFlamingo3Config()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.get_hf_config.return_value = config
|
||||
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
|
||||
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
|
||||
**data, **kwargs
|
||||
)
|
||||
ctx.model_config.hf_config = config
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def check_transformers_version():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
def test_audio_chunk_counting(mock_ctx):
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
AudioFlamingo3DummyInputsBuilder,
|
||||
AudioFlamingo3MultiModalProcessor,
|
||||
AudioFlamingo3ProcessingInfo,
|
||||
)
|
||||
|
||||
info = AudioFlamingo3ProcessingInfo(mock_ctx)
|
||||
processor = AudioFlamingo3MultiModalProcessor(
|
||||
info, AudioFlamingo3DummyInputsBuilder(info)
|
||||
)
|
||||
|
||||
sr = 16000
|
||||
audio_1 = np.zeros(30 * sr)
|
||||
audio_2 = np.zeros(75 * sr)
|
||||
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
|
||||
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
|
||||
processed = processor._apply_hf_processor_main(mm_items, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
assert chunk_counts[0].item() == 1
|
||||
assert chunk_counts[1].item() == 2
|
||||
assert len(chunk_counts) == 2
|
||||
assert processed["feature_attention_mask"].shape == (3, 3000)
|
||||
|
||||
|
||||
def test_dummy_data_generation(mock_ctx):
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
AudioFlamingo3DummyInputsBuilder,
|
||||
AudioFlamingo3ProcessingInfo,
|
||||
)
|
||||
|
||||
info = AudioFlamingo3ProcessingInfo(mock_ctx)
|
||||
builder = AudioFlamingo3DummyInputsBuilder(info)
|
||||
|
||||
mm_counts = {"audio": 2}
|
||||
dummy_data = builder.get_dummy_mm_data(100, mm_counts, MultiModalDummyOptions())
|
||||
|
||||
assert "audio" in dummy_data
|
||||
assert len(dummy_data["audio"]) == 2
|
||||
|
||||
expected_len = 60 * 16000
|
||||
assert len(dummy_data["audio"][0]) == expected_len
|
||||
|
||||
|
||||
def test_audio_token_count_matches_hf_processor_math():
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
_count_audio_tokens_from_mask,
|
||||
)
|
||||
|
||||
feature_attention_mask = torch.zeros((3, 3000), dtype=torch.long)
|
||||
feature_attention_mask[0, :2999] = 1
|
||||
feature_attention_mask[1, :2999] = 1
|
||||
feature_attention_mask[2, :1500] = 1
|
||||
chunk_counts = torch.tensor([2, 1], dtype=torch.long)
|
||||
|
||||
assert (
|
||||
_count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 0) == 1499
|
||||
)
|
||||
assert _count_audio_tokens_from_mask(feature_attention_mask, chunk_counts, 1) == 375
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Bailing prompt expansion tests without downloading a checkpoint."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from tokenizers import Tokenizer, decoders, models, pre_tokenizers
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.model_executor.models.bailing_moe_v3_vl import (
|
||||
IMAGE_PLACEHOLDER,
|
||||
IMAGE_TOKEN,
|
||||
BailingMoeV3VLMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
|
||||
from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataItems
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def processor():
|
||||
vocab = {
|
||||
token: index
|
||||
for index, token in enumerate(sorted(pre_tokenizers.ByteLevel.alphabet()))
|
||||
}
|
||||
# A newline added by the processor must merge with an existing newline.
|
||||
vocab["ĊĊ"] = len(vocab)
|
||||
backend = Tokenizer(models.BPE(vocab=vocab, merges=[("Ċ", "Ċ")]))
|
||||
backend.pre_tokenizer = pre_tokenizers.ByteLevel(
|
||||
add_prefix_space=False, use_regex=False
|
||||
)
|
||||
backend.decoder = decoders.ByteLevel()
|
||||
tokenizer = TokenizersBackend(
|
||||
tokenizer_object=backend,
|
||||
additional_special_tokens=[
|
||||
"<|vision_start|>",
|
||||
IMAGE_TOKEN,
|
||||
"<|vision_end|>",
|
||||
],
|
||||
clean_up_tokenization_spaces=False,
|
||||
)
|
||||
processor = object.__new__(BailingMoeV3VLMultiModalProcessor)
|
||||
processor.info = SimpleNamespace(
|
||||
get_tokenizer=lambda: tokenizer,
|
||||
get_hf_config=lambda: SimpleNamespace(
|
||||
vision_config=SimpleNamespace(spatial_merge_size=2)
|
||||
),
|
||||
)
|
||||
return processor
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt", "expected_text", "token_counts"),
|
||||
[
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_TOKEN}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}\nafter",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n\nafter",
|
||||
[2],
|
||||
),
|
||||
(
|
||||
f"before{IMAGE_PLACEHOLDER}{IMAGE_PLACEHOLDER}after",
|
||||
f"before<|vision_start|>{IMAGE_TOKEN * 2}<|vision_end|>\n"
|
||||
f"<|vision_start|>{IMAGE_TOKEN * 3}<|vision_end|>\nafter",
|
||||
[2, 3],
|
||||
),
|
||||
],
|
||||
ids=["wrapped", "bare", "existing_newline", "adjacent_images"],
|
||||
)
|
||||
def test_image_expansion_matches_wrappers_and_newline_tokenization(
|
||||
processor, prompt, expected_text, token_counts
|
||||
):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
image_id = tokenizer.get_vocab()[IMAGE_TOKEN]
|
||||
grids = torch.tensor([[1, 2, 2 * count] for count in token_counts])
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
{"image_grid_thw": grids},
|
||||
{"image_grid_thw": MultiModalFieldConfig.batched("image")},
|
||||
)
|
||||
mm_items = MultiModalDataItems(
|
||||
{"image": ImageProcessorItems([Image.new("RGB", (1, 1))] * len(token_counts))}
|
||||
)
|
||||
updates = processor._get_mm_prompt_updates(mm_items, {}, mm_kwargs)
|
||||
prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
|
||||
expected_ids = tokenizer.encode(expected_text, add_special_tokens=False)
|
||||
|
||||
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, updates)
|
||||
|
||||
assert actual_ids == expected_ids
|
||||
assert len(placeholders["image"]) == len(token_counts)
|
||||
found = processor._find_mm_placeholders(actual_ids, updates)
|
||||
search_offset = 0
|
||||
for index, count in enumerate(token_counts):
|
||||
placeholder = placeholders["image"][index]
|
||||
expected_offset = expected_ids.index(image_id, search_offset)
|
||||
assert placeholder.start_idx == expected_offset
|
||||
assert placeholder.tokens == [image_id] * count
|
||||
assert placeholder.to_range().get_num_embeds() == count
|
||||
assert placeholder.to_range() == found["image"][index].to_range()
|
||||
search_offset = expected_offset + count
|
||||
|
||||
|
||||
def test_text_only_prompt_preserves_original_token_ids(processor):
|
||||
tokenizer = processor.info.get_tokenizer()
|
||||
prompt_ids = tokenizer.encode("before\n", add_special_tokens=False)
|
||||
prompt_ids += tokenizer.encode("\nafter", add_special_tokens=False)
|
||||
# Deliberately use two newline tokens instead of their merged BPE token.
|
||||
assert prompt_ids != tokenizer.encode("before\n\nafter", add_special_tokens=False)
|
||||
|
||||
actual_ids, placeholders = processor._apply_prompt_updates(prompt_ids, {})
|
||||
|
||||
assert actual_ids == prompt_ids
|
||||
assert placeholders == {}
|
||||
@@ -1,244 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for DeepSeek-OCR TensorSchema validation with empty images_crop.
|
||||
|
||||
When using the Gundam preset (BASE_SIZE=1024, IMAGE_SIZE=640, CROP_MODE=True),
|
||||
images that are small enough to not require cropping produce an empty
|
||||
images_crop tensor with shape (0, 3, 640, 640). The _parse_and_validate_image_input
|
||||
method must correctly read image_size from this tensor's shape rather than
|
||||
falling back to base_size, which would cause a TensorSchema mismatch.
|
||||
|
||||
Run with:
|
||||
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.model_executor.models.deepseek_ocr import (
|
||||
DeepseekOCRForCausalLM,
|
||||
DeepseekOCRImagePixelInputs,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
DeepseekOCRProcessor,
|
||||
ImageTransform,
|
||||
)
|
||||
|
||||
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
"""Load the DeepseekOCRProcessor with tokenizer from HuggingFace."""
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
return DeepseekOCRProcessor(tokenizer=tokenizer)
|
||||
|
||||
|
||||
class TestDeepseekOCREmptyImagesCrop:
|
||||
"""Verify TensorSchema validation handles empty images_crop correctly."""
|
||||
|
||||
def test_empty_images_crop_small_image(self, processor):
|
||||
"""A small image (<=640px) produces empty images_crop and should
|
||||
not crash the TensorSchema validation.
|
||||
|
||||
Previously, the code used ``numel() > 0`` to decide whether to read
|
||||
image_size from the tensor shape. When numel()==0, it fell back to
|
||||
base_size=1024, mismatching the actual tensor dim of 640.
|
||||
"""
|
||||
# Small image: both dims <= IMAGE_SIZE (640) → no crops
|
||||
small_image = Image.new("RGB", (100, 100), color="red")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[small_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
# Processor must produce an empty crop tensor for a small image
|
||||
assert images_crop.shape[0] == 0
|
||||
|
||||
base_size = pixel_values.shape[-1]
|
||||
image_size = images_crop.shape[-1] if images_crop is not None else base_size
|
||||
|
||||
# This should NOT raise ValueError
|
||||
schema = DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": base_size,
|
||||
"image_size": image_size,
|
||||
},
|
||||
)
|
||||
|
||||
assert schema.data.shape == (1, 3, 1024, 1024)
|
||||
assert schema.images_crop.shape == (0, 3, 640, 640)
|
||||
|
||||
def test_populated_images_crop_large_image(self, processor):
|
||||
"""A large image (>640px) produces populated images_crop."""
|
||||
# Large image: exceeds IMAGE_SIZE (640) → dynamic crop tiles
|
||||
large_image = Image.new("RGB", (1200, 800), color="blue")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[large_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
assert images_crop.shape[0] > 0
|
||||
|
||||
base_size = pixel_values.shape[-1]
|
||||
image_size = images_crop.shape[-1]
|
||||
|
||||
schema = DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": base_size,
|
||||
"image_size": image_size,
|
||||
},
|
||||
)
|
||||
|
||||
assert schema.data.shape == (1, 3, 1024, 1024)
|
||||
assert schema.images_crop.shape[-1] == 640
|
||||
|
||||
def test_mismatched_image_size_raises(self, processor):
|
||||
"""Deliberately wrong image_size binding should still be caught
|
||||
by TensorSchema validation."""
|
||||
small_image = Image.new("RGB", (100, 100), color="green")
|
||||
|
||||
result = processor(
|
||||
prompt="<image>\nDescribe this image.",
|
||||
images=[small_image],
|
||||
)
|
||||
|
||||
pixel_values = result["pixel_values"]
|
||||
images_crop = result["images_crop"]
|
||||
images_spatial_crop = result["images_spatial_crop"]
|
||||
|
||||
with pytest.raises(ValueError, match="images_crop"):
|
||||
DeepseekOCRImagePixelInputs(
|
||||
type="pixel_values",
|
||||
data=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
resolve_bindings={
|
||||
"base_size": 1024,
|
||||
"image_size": 1024, # Wrong! Tensor has 640
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestDeepseekOCRInputValidation:
|
||||
"""Bare ``assert`` used to validate user input here returned HTTP 500
|
||||
(AssertionError) instead of HTTP 400 (ValueError) and disappeared under
|
||||
``python -O``, silently corrupting the tokenized sequence. See #53850.
|
||||
"""
|
||||
|
||||
def test_mismatched_image_count_raises_value_error(self, processor):
|
||||
"""Two ``<image>`` tokens with one image must raise ValueError, not
|
||||
AssertionError (which would yield HTTP 500 and, under ``-O``, silently
|
||||
drop the middle text segment)."""
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="does not match number of images"):
|
||||
processor(prompt="<image> and <image>", images=[image])
|
||||
|
||||
def test_missing_prompt_raises_value_error(self, processor):
|
||||
image = Image.new("RGB", (100, 100), color="red")
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt=None, images=[image])
|
||||
|
||||
def test_missing_images_raises_value_error(self, processor):
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt="<image>\nDescribe this image.", images=None)
|
||||
|
||||
|
||||
def _balanced_normalized_pixels(
|
||||
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
|
||||
) -> torch.Tensor:
|
||||
"""Build a pixel tensor whose values cancel to an exact sum of 0.
|
||||
|
||||
Matches the normalized half-black / half-white case from the DeepSeek-OCR
|
||||
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
|
||||
"""
|
||||
assert width % 2 == 0
|
||||
pixel_values = torch.ones(batch, channels, height, width)
|
||||
pixel_values[..., : width // 2] = -1.0
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
return pixel_values
|
||||
|
||||
|
||||
class TestDeepseekOCRZeroSumPixelsAccepted:
|
||||
"""Zero-sum normalized pixels must still produce validated image inputs.
|
||||
|
||||
Returning None from ``_parse_and_validate_image_input`` makes
|
||||
``embed_multimodal`` return None and trips the worker encoder-output
|
||||
sanity check, killing EngineCore. The zero-sum shortcut is therefore
|
||||
removed; balanced images must parse like any other valid tensor.
|
||||
"""
|
||||
|
||||
def test_ocr_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_ocr2_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
|
||||
|
||||
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
|
||||
model,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_image_transform_half_black_half_white_sums_to_zero(self):
|
||||
"""A 1024×1024 half-black/half-white PNG through ImageTransform
|
||||
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
|
||||
must still accept it.
|
||||
"""
|
||||
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
|
||||
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
|
||||
pixel_values = ImageTransform()(image).unsqueeze(0)
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
|
||||
images_crop = torch.zeros(0, 3, 640, 640)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
assert image_input is not None
|
||||
assert torch.equal(image_input.data, pixel_values)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek-V4 VL processing helpers."""
|
||||
|
||||
from PIL import Image
|
||||
from transformers import DeepseekV4Config
|
||||
|
||||
from vllm.models.deepseek_v4.common.mm_preprocess import (
|
||||
DeepseekV4VLProcessingInfo,
|
||||
DeepseekV4VLProcessor,
|
||||
build_image_block_pad_free,
|
||||
)
|
||||
|
||||
# Vision fields of deepseek-ai/DeepSeek-V4-Flash-Vision-Exp
|
||||
VISION_CONFIG = dict(
|
||||
vision_n_layers=24,
|
||||
vision_dim=1024,
|
||||
vision_n_heads=16,
|
||||
vision_inter_dim=2816,
|
||||
vision_patch_size=14,
|
||||
vision_rope_theta=10000.0,
|
||||
vision_downsample_ratio=3,
|
||||
vision_max_n_token=384,
|
||||
vision_min_pixels=147456,
|
||||
vision_max_wh_ratio=8,
|
||||
)
|
||||
|
||||
|
||||
class _ConfigCtx:
|
||||
def __init__(self, config: DeepseekV4Config) -> None:
|
||||
self._config = config
|
||||
|
||||
def get_hf_config(self, *args, **kwargs):
|
||||
return self._config
|
||||
|
||||
|
||||
def _feature_counts(
|
||||
processor: DeepseekV4VLProcessor, width: int, height: int
|
||||
) -> tuple[int, int]:
|
||||
out = processor(images=[Image.new("RGB", (width, height))])
|
||||
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
|
||||
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
|
||||
return len(types), out["patches"].shape[0]
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_beats_wide_real_image():
|
||||
"""Dummy size must maximize tokens/patches within the budget (#59271).
|
||||
|
||||
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
|
||||
"""
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
processor = DeepseekV4VLProcessor(config)
|
||||
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
|
||||
wide_features = _feature_counts(processor, 1932, 336)
|
||||
|
||||
assert dummy == (1932, 336)
|
||||
assert wide_features <= dummy_features
|
||||
|
||||
|
||||
def test_get_image_size_with_most_features_respects_max_wh_ratio():
|
||||
config = DeepseekV4Config(**VISION_CONFIG)
|
||||
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
|
||||
dummy = info.get_image_size_with_most_features()
|
||||
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
|
||||
@@ -1,189 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.gemma3n_audio_utils import (
|
||||
adjust_audio_features_to_expected_length,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# Gemma3 (image) model
|
||||
GEMMA3_MODEL_ID = "google/gemma-3-4b-it"
|
||||
|
||||
# Gemma3n (multimodal with audio) model
|
||||
GEMMA3N_MODEL_ID = "google/gemma-3n-E2B-it"
|
||||
|
||||
# Expected audio tokens for Gemma3n (audio_soft_tokens_per_image)
|
||||
GEMMA3N_EXPECTED_AUDIO_TOKENS = 188
|
||||
|
||||
|
||||
class TestGemma3nAudioTensorLogic:
|
||||
"""CPU-based tests for Gemma3n audio feature tensor manipulation.
|
||||
|
||||
These tests validate the padding/truncation logic in
|
||||
adjust_audio_features_to_expected_length() which fixes the
|
||||
integer overflow in _process_audio_input when audio_seq_len > 188.
|
||||
"""
|
||||
|
||||
def test_padding_when_audio_short(self):
|
||||
"""Test that short audio is padded to expected length."""
|
||||
batch_size, seq_len, embed_dim = 1, 100, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == 0
|
||||
# First 100 tokens should be original, rest should be padding (zeros)
|
||||
assert torch.allclose(result[:, :seq_len, :], audio_features)
|
||||
assert torch.allclose(
|
||||
result[:, seq_len:, :],
|
||||
torch.zeros(batch_size, expected_tokens - seq_len, embed_dim),
|
||||
)
|
||||
|
||||
def test_truncation_when_audio_long(self):
|
||||
"""Test that long audio is truncated to expected length.
|
||||
|
||||
This is the key test for the overflow fix. Previously, when
|
||||
audio_seq_len > expected_tokens, the code would compute a negative
|
||||
padding value causing: RuntimeError: numel: integer multiplication overflow
|
||||
"""
|
||||
batch_size, seq_len, embed_dim = 1, 192, 256 # 192 > 188
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == seq_len - expected_tokens # 192 - 188 = 4
|
||||
# Result should be first 188 tokens of original
|
||||
assert torch.allclose(result, audio_features[:, :expected_tokens, :])
|
||||
|
||||
def test_no_change_when_exact_length(self):
|
||||
"""Test that exact-length audio passes through unchanged."""
|
||||
batch_size, embed_dim = 1, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, expected_tokens, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == audio_features.shape
|
||||
assert tokens_truncated == 0
|
||||
assert torch.allclose(result, audio_features)
|
||||
|
||||
def test_original_bug_would_fail(self):
|
||||
"""Verify the original buggy implementation would cause overflow.
|
||||
|
||||
The original code always tried to pad, which fails when
|
||||
audio_seq_len > expected_tokens because expand() gets negative size.
|
||||
"""
|
||||
batch_size, seq_len, embed_dim = 1, 192, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
# Original buggy logic (always pads, never truncates)
|
||||
extra_padding_tokens = expected_tokens - seq_len # = -4 (negative!)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
# This should fail with negative size error
|
||||
padding_embs.expand(batch_size, extra_padding_tokens, embed_dim)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_len",
|
||||
[50, 100, 150, 187, 188, 189, 192, 200, 300],
|
||||
)
|
||||
def test_various_audio_lengths(self, seq_len: int):
|
||||
"""Test padding/truncation with various audio lengths."""
|
||||
batch_size, embed_dim = 1, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
# Should not raise any errors
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
# Output should always be expected_tokens length
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
|
||||
# Verify truncation count is correct
|
||||
if seq_len > expected_tokens:
|
||||
assert tokens_truncated == seq_len - expected_tokens
|
||||
else:
|
||||
assert tokens_truncated == 0
|
||||
|
||||
def test_batch_processing(self):
|
||||
"""Test that batch processing works correctly."""
|
||||
batch_size, seq_len, embed_dim = 4, 192, 256
|
||||
expected_tokens = GEMMA3N_EXPECTED_AUDIO_TOKENS
|
||||
|
||||
audio_features = torch.randn(batch_size, seq_len, embed_dim)
|
||||
padding_embs = torch.zeros(1, 1, embed_dim)
|
||||
|
||||
result, tokens_truncated = adjust_audio_features_to_expected_length(
|
||||
audio_features, expected_tokens, padding_embs
|
||||
)
|
||||
|
||||
assert result.shape == (batch_size, expected_tokens, embed_dim)
|
||||
assert tokens_truncated == seq_len - expected_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA3_MODEL_ID])
|
||||
@pytest.mark.parametrize("mm_processor_kwargs", [{}])
|
||||
def test_get_image_size_with_most_features(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
|
||||
|
||||
max_image_size = processor.info.get_image_size_with_most_features()
|
||||
max_tokens = processor.info.get_num_image_tokens(
|
||||
image_width=max_image_size.width,
|
||||
image_height=max_image_size.height,
|
||||
processor=hf_processor,
|
||||
mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
|
||||
prompt = "<start_of_image>"
|
||||
image_seq_length = hf_processor.image_seq_length
|
||||
|
||||
for asset in image_assets:
|
||||
mm_data = {"image": [asset.pil_image]}
|
||||
processed_inputs = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
mm_kwargs_data = processed_inputs["mm_kwargs"].get_data()
|
||||
num_patches_tensor = mm_kwargs_data["num_patches"]
|
||||
tokens = int(num_patches_tensor.item()) * image_seq_length
|
||||
assert tokens <= max_tokens
|
||||
@@ -1,405 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ForConditionalGeneration,
|
||||
Gemma4ImagePixelInputs,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
|
||||
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
|
||||
|
||||
|
||||
def test_gemma4_image_schema_accepts_variable_patch_counts():
|
||||
Gemma4ImagePixelInputs(
|
||||
pixel_values=[
|
||||
torch.randn(10080, 768),
|
||||
torch.randn(2520, 768),
|
||||
],
|
||||
pixel_position_ids=[
|
||||
torch.zeros(10080, 2, dtype=torch.long),
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_image_batching_keeps_variable_patch_counts_unstacked():
|
||||
field = MultiModalFieldConfig.batched("image").field
|
||||
elems = field.build_elems(
|
||||
"image",
|
||||
"pixel_values",
|
||||
[torch.randn(10080, 768), torch.randn(2520, 768)],
|
||||
)
|
||||
|
||||
reduced = field.reduce_data(list(elems))
|
||||
|
||||
assert isinstance(reduced, list)
|
||||
assert [tensor.shape for tensor in reduced] == [
|
||||
torch.Size([10080, 768]),
|
||||
torch.Size([2520, 768]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_width,image_height,max_soft_tokens",
|
||||
[
|
||||
# Production repro: a 3x900 image (extreme aspect ratio) made the
|
||||
# prompt-side estimator return 289 while the HF Gemma 4 image
|
||||
# processor's vision tower output capped at 280, producing the
|
||||
# "Attempted to assign 280 multimodal tokens to 289 placeholders"
|
||||
# mismatch that crashed EngineCore.
|
||||
(900, 3, 280),
|
||||
(3, 900, 280),
|
||||
# Same pathology should hold for the video-frame budget (70 tokens).
|
||||
(900, 3, 70),
|
||||
# And for any other supported budget.
|
||||
(4000, 2, 1120),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
|
||||
model_id: str,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
max_soft_tokens: int,
|
||||
):
|
||||
"""Regression for the Gemma 3/4 multimodal crash.
|
||||
|
||||
`_compute_num_soft_tokens` must never return a value larger than
|
||||
`max_soft_tokens`. The HF Gemma 4 image processor clamps its vision
|
||||
tower output to that value; if the prompt-side estimator returns more,
|
||||
the prompt has more `image` placeholder tokens than the encoder will
|
||||
fill, and `_merge_multimodal_embeddings` raises `ValueError` deep in
|
||||
the model forward.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
num_soft_tokens = processor.info._compute_num_soft_tokens(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
max_soft_tokens=max_soft_tokens,
|
||||
)
|
||||
|
||||
assert num_soft_tokens <= max_soft_tokens, (
|
||||
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
|
||||
f"image_width={image_width}, image_height={image_height}, "
|
||||
f"max_soft_tokens={max_soft_tokens} — exceeds the cap that the HF "
|
||||
f"image processor enforces on its vision tower output. This is "
|
||||
f"the placeholder/encoder count mismatch that crashes EngineCore."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_image_tokens"),
|
||||
[
|
||||
({}, 280),
|
||||
({"max_soft_tokens": 70}, 70),
|
||||
({"max_soft_tokens": 280}, 280),
|
||||
({"max_soft_tokens": 1120}, 1120),
|
||||
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
|
||||
({"images_kwargs": None}, 280),
|
||||
({"images_kwargs": "not-a-dict"}, 280),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_image_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"image": 1, "video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == expected_image_tokens
|
||||
assert tokens["video"] == 32 * (70 + 2 + 6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit_mm_per_prompt", "expected_video_tokens"),
|
||||
[
|
||||
({"video": 1}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
|
||||
model_id: str,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
|
||||
expected_video_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == 280
|
||||
assert tokens["video"] == expected_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
image = PILImage.new("RGB", (1000, 1000), color="white")
|
||||
image_size = image.size
|
||||
mm_items = processor.info.parse_mm_data({"image": image})
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
replacement = prompt_update.resolve(0).content.full
|
||||
expected = processor.info.get_image_repl(
|
||||
image_width=image_size[0],
|
||||
image_height=image_size[1],
|
||||
processor=processor.info.get_hf_processor(),
|
||||
max_soft_tokens=560,
|
||||
).full
|
||||
|
||||
assert replacement == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
@pytest.mark.parametrize(
|
||||
"image_kwargs", [{"rescale_factor": 1 / 127.5}, {"max_soft_tokens": 560}]
|
||||
)
|
||||
@pytest.mark.parametrize("video_uuid", [None, "same-video"])
|
||||
def test_video_cache_is_independent_of_image_kwargs(
|
||||
model_id: str,
|
||||
image_kwargs: dict[str, object],
|
||||
video_uuid: str | None,
|
||||
):
|
||||
"""Request image overrides must not change video frames or make results
|
||||
depend on cache warmth.
|
||||
"""
|
||||
request_kwargs = {"images_kwargs": image_kwargs}
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
mm_processor_cache_gb=1,
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(ctx.model_config)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
hf_processor = processor.info.get_hf_processor()
|
||||
image = PILImage.new("RGB", (48, 48), color=(128, 128, 128))
|
||||
frames = np.stack([np.asarray(image)] * 2)
|
||||
metadata = {"fps": 2.0, "frames_indices": [0, 1]}
|
||||
mm_items = processor.info.parse_mm_data(
|
||||
{"image": image, "video": [(frames, metadata)]}
|
||||
)
|
||||
|
||||
def process(mm_kwargs, cache):
|
||||
return processor(
|
||||
hf_processor.image_token + hf_processor.video_token,
|
||||
mm_items,
|
||||
mm_uuid_items={"video": [video_uuid]},
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
# Populate the video cache without image overrides, then change only
|
||||
# images_kwargs on the next request. Since images_kwargs are excluded from
|
||||
# the video hash, and must not affect the processed video, the cached result
|
||||
# must match a fresh recomputation.
|
||||
baseline = process({}, cache=cache)
|
||||
cached = process(request_kwargs, cache=cache)
|
||||
cache.clear_cache()
|
||||
fresh = process(request_kwargs, cache=cache)
|
||||
|
||||
def pixels(result, modality, field):
|
||||
return result["mm_kwargs"][modality][0][field].data
|
||||
|
||||
video_pixels = pixels(fresh, "video", "pixel_values_videos")
|
||||
assert torch.equal(pixels(cached, "video", "pixel_values_videos"), video_pixels)
|
||||
assert torch.equal(pixels(baseline, "video", "pixel_values_videos"), video_pixels)
|
||||
assert baseline["mm_hashes"]["video"] == fresh["mm_hashes"]["video"]
|
||||
assert not torch.equal(
|
||||
pixels(baseline, "image", "pixel_values"),
|
||||
pixels(fresh, "image", "pixel_values"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_limit_mm_per_prompt(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
|
||||
# We only allow 1 image
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Provide 2 images in the prompt
|
||||
prompt = "<image><image>"
|
||||
# image_assets usually has multiple images
|
||||
images = [asset.pil_image for asset in image_assets][:2]
|
||||
if len(images) < 2:
|
||||
images = [images[0], images[0]]
|
||||
|
||||
mm_data = {"image": images}
|
||||
|
||||
# Expect ValueError when exceeding limit
|
||||
with pytest.raises(VLLMValidationError, match="At most 1 image"):
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
# Regression guard for PR #43169 follow-up: the batched Gemma4 vision encoder
|
||||
# admitted ``chunk ~= 53`` on a 22 GiB L4 with a 26B AWQ model loaded,
|
||||
# allocating 2.43 GiB int64 inside
|
||||
# ``F.one_hot(num_classes=position_embedding_size)`` and OOMing because only
|
||||
# 2.41 GiB was actually free. The fix sizes ``chunk`` from currently-free GPU
|
||||
# memory and counts the ``F.one_hot`` transient as the dominant cost.
|
||||
|
||||
_encoder_chunk = Gemma4ForConditionalGeneration._encoder_chunk
|
||||
|
||||
# Gemma4 vision config default (HF: configuration_gemma4.py).
|
||||
_POSITION_EMBEDDING_SIZE = 10240
|
||||
# Video frame: max_soft_tokens=70, pooling_kernel_size=2 -> 70 * 4 patches.
|
||||
_VIDEO_PATCHES_PER_FRAME = 280
|
||||
|
||||
|
||||
def test_encoder_chunk_tight_budget_fits_in_free():
|
||||
free = 3 * GiB_bytes # L4 22 GiB after 26B AWQ load.
|
||||
total = 22 * GiB_bytes
|
||||
chunk = _encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME, free, total, _POSITION_EMBEDDING_SIZE
|
||||
)
|
||||
one_hot_bytes = chunk * _VIDEO_PATCHES_PER_FRAME * 2 * _POSITION_EMBEDDING_SIZE * 8
|
||||
assert one_hot_bytes <= free // 2
|
||||
|
||||
|
||||
def test_encoder_chunk_roomy_gpu_keeps_batching():
|
||||
chunk = _encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME,
|
||||
60 * GiB_bytes,
|
||||
80 * GiB_bytes,
|
||||
_POSITION_EMBEDDING_SIZE,
|
||||
)
|
||||
assert chunk > 8
|
||||
|
||||
|
||||
def test_encoder_chunk_zero_patches_is_safe():
|
||||
assert (
|
||||
_encoder_chunk(0, 60 * GiB_bytes, 80 * GiB_bytes, _POSITION_EMBEDDING_SIZE) == 1
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_chunk_zero_position_embedding_size_is_safe():
|
||||
# Degenerate config: must not raise ZeroDivisionError.
|
||||
assert (
|
||||
_encoder_chunk(_VIDEO_PATCHES_PER_FRAME, 60 * GiB_bytes, 80 * GiB_bytes, 0) == 1
|
||||
)
|
||||
|
||||
|
||||
def test_encoder_chunk_no_free_memory_falls_back_to_one():
|
||||
assert (
|
||||
_encoder_chunk(
|
||||
_VIDEO_PATCHES_PER_FRAME, 0, 22 * GiB_bytes, _POSITION_EMBEDDING_SIZE
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
def _gemma4_for_encoder_cudagraph() -> Gemma4ForConditionalGeneration:
|
||||
model = object.__new__(Gemma4ForConditionalGeneration)
|
||||
vision_config = SimpleNamespace(
|
||||
patch_size=16, pooling_kernel_size=3, default_output_length=280
|
||||
)
|
||||
model.vision_tower = SimpleNamespace(config=vision_config)
|
||||
model.config = SimpleNamespace(vision_config=vision_config)
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("token_budget", "row_tokens"), [(4096, 70), (4096, 280), (70, 280)]
|
||||
)
|
||||
def test_encoder_cudagraph_capture_rows_fit_token_budget(
|
||||
token_budget: int, row_tokens: int
|
||||
):
|
||||
# Regression for #61045: rows were max_frames_per_batch (58 * 32) at
|
||||
# 280 tokens each, so the 4096-token buffer alone was 6.69 GiB.
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
inputs = model.prepare_encoder_cudagraph_capture_inputs(
|
||||
token_budget=token_budget,
|
||||
max_batch_size=58,
|
||||
max_frames_per_batch=58 * 32,
|
||||
device="meta",
|
||||
dtype=torch.bfloat16,
|
||||
axis_keys=(row_tokens,),
|
||||
)
|
||||
|
||||
num_rows, row_patches, _ = inputs.values["pixel_values"].shape
|
||||
assert row_patches == row_tokens * 9
|
||||
assert num_rows == max(1, token_budget // row_tokens)
|
||||
|
||||
|
||||
def test_encoder_cudagraph_mixed_width_images_pack_by_padded_width():
|
||||
model = _gemma4_for_encoder_cudagraph()
|
||||
mm_kwargs = {
|
||||
"pixel_values": [torch.zeros(2520, 768), torch.zeros(630, 768)],
|
||||
"pixel_position_ids": [
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
torch.zeros(630, 2, dtype=torch.long),
|
||||
],
|
||||
}
|
||||
|
||||
specs = model.get_encoder_cudagraph_item_specs(mm_kwargs)
|
||||
assert [spec.output_tokens for spec in specs] == [280, 70]
|
||||
assert [spec.get_path_output_tokens("default") for spec in specs] == [280, 280]
|
||||
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (70,)
|
||||
selected = model.select_encoder_cudagraph_items(mm_kwargs, [0, 1])
|
||||
assert selected[ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG] == (280,)
|
||||
@@ -1,248 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image as PILImage
|
||||
from transformers.models.gemma4 import Gemma4AudioFeatureExtractor
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4ImagePixelInputs,
|
||||
_dummy_audio_num_samples,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFieldConfig
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# The Unified model ID for testing purposes
|
||||
GEMMA4_UNIFIED_MODEL_ID = "google/gemma-4-12B-it"
|
||||
|
||||
|
||||
def test_gemma4_unified_image_schema_accepts_variable_patch_counts():
|
||||
Gemma4ImagePixelInputs(
|
||||
pixel_values=[
|
||||
torch.randn(10080, 768),
|
||||
torch.randn(2520, 768),
|
||||
],
|
||||
pixel_position_ids=[
|
||||
torch.zeros(10080, 2, dtype=torch.long),
|
||||
torch.zeros(2520, 2, dtype=torch.long),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_gemma4_unified_image_batching_keeps_variable_patch_counts_unstacked():
|
||||
field = MultiModalFieldConfig.batched("image").field
|
||||
elems = field.build_elems(
|
||||
"image",
|
||||
"pixel_values",
|
||||
[torch.randn(10080, 768), torch.randn(2520, 768)],
|
||||
)
|
||||
|
||||
reduced = field.reduce_data(list(elems))
|
||||
|
||||
assert isinstance(reduced, list)
|
||||
assert [tensor.shape for tensor in reduced] == [
|
||||
torch.Size([10080, 768]),
|
||||
torch.Size([2520, 768]),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"image_width,image_height,max_soft_tokens",
|
||||
[
|
||||
(900, 3, 280),
|
||||
(3, 900, 280),
|
||||
(900, 3, 70),
|
||||
(4000, 2, 1120),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_compute_num_soft_tokens_does_not_exceed_max_soft_tokens(
|
||||
model_id: str,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
max_soft_tokens: int,
|
||||
):
|
||||
"""Verify ``_compute_num_soft_tokens`` caps output at ``max_soft_tokens``."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"do_pan_and_scan": True},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
num_soft_tokens = processor.info._compute_num_soft_tokens(
|
||||
image_width=image_width,
|
||||
image_height=image_height,
|
||||
max_soft_tokens=max_soft_tokens,
|
||||
)
|
||||
|
||||
assert num_soft_tokens <= max_soft_tokens, (
|
||||
f"_compute_num_soft_tokens returned {num_soft_tokens} for "
|
||||
f"image_width={image_width}, image_height={image_height}, "
|
||||
f"max_soft_tokens={max_soft_tokens} — exceeds the cap."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mm_processor_kwargs", "expected_image_tokens"),
|
||||
[
|
||||
({}, 280),
|
||||
({"max_soft_tokens": 70}, 70),
|
||||
({"max_soft_tokens": 280}, 280),
|
||||
({"max_soft_tokens": 1120}, 1120),
|
||||
({"images_kwargs": {"max_soft_tokens": 560}}, 560),
|
||||
({"images_kwargs": None}, 280),
|
||||
({"images_kwargs": "not-a-dict"}, 280),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_max_soft_tokens(
|
||||
model_id: str,
|
||||
mm_processor_kwargs: dict[str, object],
|
||||
expected_image_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt={"image": 1, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"image": 1, "video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == expected_image_tokens
|
||||
assert tokens["video"] == 32 * (70 + 2 + 6)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("limit_mm_per_prompt", "expected_video_tokens"),
|
||||
[
|
||||
({"video": 1}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 1}}, 1 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 8}}, 8 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 32}}, 32 * (70 + 2 + 6)),
|
||||
({"video": {"count": 1, "num_frames": 40}}, 32 * (70 + 2 + 6)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_mm_max_tokens_per_item_respects_configured_video_num_frames(
|
||||
model_id: str,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]],
|
||||
expected_video_tokens: int,
|
||||
):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
tokens = processor.info.get_mm_max_tokens_per_item(
|
||||
seq_len=ctx.model_config.max_model_len,
|
||||
mm_counts={"video": 1},
|
||||
)
|
||||
|
||||
assert tokens is not None
|
||||
assert tokens["image"] == 280
|
||||
assert tokens["video"] == expected_video_tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_get_prompt_updates_respects_nested_max_soft_tokens(model_id: str):
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={"images_kwargs": {"max_soft_tokens": 560}},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
image = PILImage.new("RGB", (1000, 1000), color="white")
|
||||
image_size = image.size
|
||||
mm_items = processor.info.parse_mm_data({"image": image})
|
||||
|
||||
prompt_update = processor._get_prompt_updates(mm_items, {}, {})[0]
|
||||
replacement = prompt_update.resolve(0).content.full
|
||||
expected = processor.info.get_image_repl(
|
||||
image_width=image_size[0],
|
||||
image_height=image_size[1],
|
||||
processor=processor.info.get_hf_processor(),
|
||||
max_soft_tokens=560,
|
||||
).full
|
||||
|
||||
assert replacement == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_UNIFIED_MODEL_ID])
|
||||
def test_limit_mm_per_prompt(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that limit_mm_per_prompt restricts multiple images correctly."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<image><image>"
|
||||
images = [asset.pil_image for asset in image_assets][:2]
|
||||
if len(images) < 2:
|
||||
images = [images[0], images[0]]
|
||||
|
||||
mm_data = {"image": images}
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="At most 1 image"):
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
|
||||
def test_dummy_audio_length_falls_back_for_the_unified_extractor():
|
||||
"""The unified audio feature extractor does not define ``fft_length``.
|
||||
|
||||
``Gemma4DummyInputsBuilder.get_dummy_mm_data`` read that attribute
|
||||
unconditionally, so memory profiling raised ``AttributeError`` for every
|
||||
unified Gemma 4 deployment in which audio is the modality with the largest
|
||||
per-item token count -- which is the case as soon as ``audio_seq_length``
|
||||
is raised above the video budget.
|
||||
"""
|
||||
|
||||
class _UnifiedFeatureExtractor:
|
||||
sampling_rate = 16000
|
||||
|
||||
class _UnifiedProcessor:
|
||||
feature_extractor = _UnifiedFeatureExtractor()
|
||||
audio_seq_length = 3000
|
||||
audio_ms_per_token = 40
|
||||
|
||||
# 3000 tokens * 40 ms = 120 s at 16 kHz
|
||||
assert _dummy_audio_num_samples(_UnifiedProcessor()) == 120 * 16000
|
||||
|
||||
|
||||
def test_dummy_audio_length_still_uses_fft_length_for_the_tower_extractor():
|
||||
"""The mel/tower extractor keeps its existing behaviour."""
|
||||
|
||||
class _TowerFeatureExtractor(Gemma4AudioFeatureExtractor):
|
||||
def __init__(self): # noqa: D107 - bypass the real feature-extractor init
|
||||
self.fft_length = 512
|
||||
self.sampling_rate = 16000
|
||||
|
||||
class _TowerProcessor:
|
||||
feature_extractor = _TowerFeatureExtractor()
|
||||
audio_seq_length = 750
|
||||
audio_ms_per_token = 40
|
||||
|
||||
assert _dummy_audio_num_samples(_TowerProcessor()) == 512
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user