Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
+170 -4
View File
@@ -5,8 +5,7 @@
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
token-strided rather than contiguous. Both backends must match a pure-PyTorch
recurrence. CUDA reads these slices in place and rejects unsupported layouts;
the ROCm wrapper makes contiguous copies before launching its kernel.
recurrence, read these slices in place and reject unsupported layouts.
"""
import pytest
@@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Only the CUDA implementation requires zero-copy token-strided layouts",
not current_platform.is_cuda_alike(),
reason="Both implementations require zero-copy token-strided layouts",
)
@torch.inference_mode()
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
@@ -193,3 +192,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts():
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
with pytest.raises(AssertionError, match=r"torch.Size"):
run_kernel(broken, state)
@pytest.mark.parametrize("offset", [16, 144, 256])
@pytest.mark.parametrize("dim_first", [False, True])
@pytest.mark.parametrize("num_spec", [0, 3])
@torch.inference_mode()
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
"""Restoring both cached states must reproduce an uninterrupted prefill."""
from types import SimpleNamespace
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.kda_checkpoint import (
FlashKDAPrefillCheckpointExporter,
)
from vllm.models.glm5next.common import kda
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
from vllm.v1.worker.workspace import WorkspaceManager
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
pytest.skip("requires FlashKDA")
import vllm._flashkda_C # noqa: F401
torch.manual_seed(42)
device = torch.device("cuda")
length, heads = 273, 2
width, state_len = 3 * heads * D, 3 + num_spec
layer = object.__new__(kda.Glm5NextLinearAttention)
torch.nn.Module.__init__(layer)
layer.prefix = "layer.0"
layer.local_num_heads = heads
layer.head_dim = D
layer.conv_size = 4
layer.local_projection_size = heads * D
layer.kda_safe_gate = True
layer.kda_lower_bound = -5.0
layer.kda_prefill_backend = "flashkda"
layer._conv_state_dim_first = dim_first
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
layer.q_conv1d = SimpleNamespace(bias=None)
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
state_shape = (2, heads, D, D)
layer._flashkda_buffer_specs = (
(state_shape, torch.float32),
(state_shape, torch.float32),
(
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
torch.uint8,
),
((1, 2 * length, heads, D), torch.bfloat16),
)
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
workspace = WorkspaceManager(device)
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
conv_shape = (width, state_len) if dim_first else (state_len, width)
num_slots = 5 + (num_spec + 1 if num_spec else 0)
conv_storage = torch.zeros(
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
)
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
recurrent_storage = torch.zeros(
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
)
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
layer.kv_cache = (conv, recurrent)
projected = torch.randn(
2 * length, width + heads, device=device, dtype=torch.bfloat16
)
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
raw_before = raw_qkv.clone()
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
def run(qkv, g, b, slots, has_initial, checkpoint=None):
n = qkv.shape[0] // 2
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
cu_cpu, device=device
)
metadata = GDNAttentionMetadata(
num_prefills=2,
num_prefill_tokens=2 * n,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=0,
num_spec_decode_tokens=0,
num_actual_tokens=2 * n,
has_initial_state=torch.tensor(has_initial, device=device),
non_spec_query_start_loc=cu_cpu.to(device),
non_spec_state_indices_tensor=torch.tensor(
slots, dtype=torch.int32, device=device
),
checkpoint=checkpoint,
nums_dict=nums,
batch_ptr=batch_ptr,
token_chunk_offset_ptr=chunk_offsets,
)
monkeypatch.setattr(
kda,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
)
spec_tokens = num_spec + 1 if num_spec else 0
if spec_tokens:
qkv = torch.cat([qkv[:spec_tokens], qkv])
g = torch.cat([g[:, :spec_tokens], g], dim=1)
b = torch.cat([b[:, :spec_tokens], b], dim=1)
metadata.num_spec_decodes = 1
metadata.num_spec_decode_tokens = spec_tokens
metadata.num_actual_tokens += spec_tokens
metadata.spec_sequence_masks = torch.tensor(
[True, False, False], device=device
)
metadata.spec_query_start_loc = torch.tensor(
[0, spec_tokens], dtype=torch.int32, device=device
)
metadata.spec_state_indices_tensor = torch.arange(
5, num_slots, dtype=torch.int32, device=device
).unsqueeze(0)
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
metadata.non_spec_token_indx = torch.arange(
spec_tokens, qkv.shape[0], device=device
)
metadata.num_accepted_tokens = torch.ones(
1, dtype=torch.int32, device=device
)
out = torch.empty(
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
)
layer._forward(qkv, g, b, out)
return out[:, spec_tokens:]
checkpoint = MambaPrefillCheckpointMetadata(
torch.tensor([offset, 0], dtype=torch.int32, device=device),
torch.tensor([3, 0], dtype=torch.int32, device=device),
)
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
expected_state = recurrent[1].clone()
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
torch.testing.assert_close(raw_qkv, raw_before)
conv_view = conv if dim_first else conv.transpose(-1, -2)
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
run(
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
[1, 2],
[False, False],
)
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
suffix = slice(offset, length)
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
conv[4].copy_(saved_conv)
recurrent[4].copy_(saved_recurrent)
actual = run(qkv, g, b, [3, 4], [True, True])
torch.testing.assert_close(
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
)
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
@@ -15,11 +15,11 @@ from types import SimpleNamespace
import pytest
import torch
from torch import nn
from transformers import Glm5NextTextConfig
from vllm.model_executor import parameter
from vllm.model_executor.layers import linear
from vllm.models.glm5next.common import model as glm_model
from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig
HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2
@@ -0,0 +1,521 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm KDA prefill checkpoint: where it is taken and what it stores.
Two things have to line up for a checkpoint to be safe: the metadata must name
a position the chunk kernel will actually write, and the conv window saved
there must be the one a resumed prefill would have left.
"""
import pytest
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention
from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn
from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDAMetadataBuilder
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
)
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
KDA_CHECKPOINT_ALIGNMENT,
is_fused_kda_chunk_supported,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
pytestmark = pytest.mark.skipif(
not (
current_platform.is_rocm()
and torch.cuda.is_available()
and is_fused_kda_chunk_supported()
),
reason="The ROCm KDA prefill checkpoint is exported by the fused chunk "
"kernel, which needs a gfx950 build that includes it",
)
# A mamba block size the chunk walk lands on. The real K3 value is 768; any
# multiple of the kernel's 64-token chunk behaves the same and keeps the
# fixtures small.
BLOCK_SIZE = 64
class _StubSpeculativeConfig:
method = "dspark"
parallel_drafting = False
def __init__(self, num_speculative_tokens: int) -> None:
self.num_speculative_tokens = num_speculative_tokens
def use_eagle_block_drop(self) -> bool:
return False
def _builder(
num_prefill_checkpoint_blocks: int = 1,
mamba_cache_mode: str = "align",
block_size: int = BLOCK_SIZE,
prefix_match_unit: int | None = None,
num_spec: int = 0,
) -> KimiK3ROCmKDAMetadataBuilder:
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B", block_size=block_size
)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
if num_spec:
vllm_config.speculative_config = _StubSpeculativeConfig(num_spec)
return KimiK3ROCmKDAMetadataBuilder(
kv_cache_spec=MambaSpec(
block_size=block_size,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=(
KDA_CHECKPOINT_ALIGNMENT if num_prefill_checkpoint_blocks else None
),
num_speculative_blocks=num_spec,
),
layer_names=["layer.0"],
vllm_config=vllm_config,
device=torch.device("cuda"),
)
def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw):
common = create_common_attn_metadata(
batch, block_size, torch.device("cuda"), arange_block_indices=True
)
return _builder(block_size=block_size, **kw).build(0, common)
def _i32(values: list[int]) -> torch.Tensor:
return torch.tensor(values, dtype=torch.int32, device="cuda")
def test_checkpoint_metadata_targets_last_aligned_boundary() -> None:
# Slot p holds the state after (p + 1) * block_size tokens, so a 200-token
# prompt checkpoints at 192 into slot 2 and a 128-token one at 64 into slot
# 0. The block table is an arange, so request 1 owns blocks 4..7.
md = _build(BatchSpec(seq_lens=[200, 128], query_lens=[200, 128]))
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 64]))
torch.testing.assert_close(md.checkpoint.state_indices, _i32([2, 4]))
def test_checkpoint_offset_is_relative_to_query_start() -> None:
# 256 tokens computed, 144 more this step: the boundary at 384 lands 128
# tokens into the chunk, and cdiv(400, 64) - 2 = 5 is its slot.
md = _build(BatchSpec(seq_lens=[400, 64], query_lens=[144, 64]))
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([128, 0]))
torch.testing.assert_close(md.checkpoint.state_indices, _i32([5, NULL_BLOCK_ID]))
def test_checkpoint_metadata_avoids_live_state_row() -> None:
# Writing the row the walk itself owns would race it.
md = _build(BatchSpec(seq_lens=[200, 130], query_lens=[200, 130]))
assert md.checkpoint is not None
# `mamba_get_block_table_tensor` hands the walk block (seq_len - 1) // bs.
live = torch.tensor([(200 - 1) // BLOCK_SIZE, (130 - 1) // BLOCK_SIZE])
table = torch.arange(2 * 4).view(2, 4)
live_rows = table[torch.arange(2), live].cuda()
assert not bool((md.checkpoint.state_indices == live_rows).any())
def test_checkpoint_metadata_skips_chunk_without_boundary() -> None:
# 128 computed, 40 more: the only boundary at 128 is the state this chunk
# started from, so there is nothing new to snapshot.
assert _build(BatchSpec(seq_lens=[168], query_lens=[40])).checkpoint is None
def test_checkpoint_metadata_skips_null_reservation() -> None:
# Only the chunk that finishes a prompt gets a block reserved; every
# earlier one leaves the shared null block, which nothing hashes and
# writing it would corrupt every other request's scratch row.
batch = BatchSpec(seq_lens=[200], query_lens=[200])
common = create_common_attn_metadata(
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
)
# cdiv(200, 64) - 2 = 2 is the checkpoint slot.
common.block_table_tensor[0, 2] = NULL_BLOCK_ID
md = _builder().build(0, common)
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.state_indices, _i32([NULL_BLOCK_ID]))
def test_checkpoint_metadata_absent_without_reserved_block() -> None:
md = _build(
BatchSpec(seq_lens=[200], query_lens=[200]),
num_prefill_checkpoint_blocks=0,
)
assert getattr(md, "checkpoint", None) is None
def test_spec_checkpoint_metadata_gathers_non_spec_rows_by_index() -> None:
# gdn_attn selects the non-spec rows by mask and reclassifies their
# one-token decodes as prefills, so the chunk kernel is handed rows 1, 3
# and 4 here while a num_decodes: slice would hand it rows 0, 1 and 2.
num_spec = 2
batch = BatchSpec(
seq_lens=[300, 200, 400, 512, 128],
query_lens=[num_spec + 1, 200, num_spec + 1, 1, 128],
)
common = create_common_attn_metadata(
batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True
)
# Align mode reads 1 + num_speculative_blocks columns, so widen the table.
rows, cols = common.block_table_tensor.shape
width = cols + num_spec
common.block_table_tensor = torch.arange(
rows * width, dtype=torch.int32, device="cuda"
).view(rows, width)
md = _builder(num_spec=num_spec).build(
0,
common,
num_accepted_tokens=torch.ones(rows, dtype=torch.int32, device="cuda"),
num_decode_draft_tokens_cpu=torch.tensor(
[num_spec, -1, num_spec, -1, -1], dtype=torch.int32
),
)
assert md.spec_sequence_masks is not None
assert md.num_decodes == 0 and md.num_prefills == 3
assert md.checkpoint is not None
# Row 3 resumes mid-block, so its chunk holds no reusable boundary.
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 0, 64]))
# Request r owns blocks r*width.., so slot cdiv(200, 64) - 2 = 2 for
# request 1 and cdiv(128, 64) - 2 = 0 for request 4.
torch.testing.assert_close(
md.checkpoint.state_indices, _i32([width + 2, NULL_BLOCK_ID, 4 * width])
)
def test_checkpoint_metadata_absent_outside_align_mode() -> None:
md = _build(
BatchSpec(seq_lens=[200], query_lens=[200]),
mamba_cache_mode="none",
)
assert getattr(md, "checkpoint", None) is None
# ---------------------------------------------------------------------------
# Conv half
# ---------------------------------------------------------------------------
DIM = 384
WIDTH = 4
STATE_LEN = WIDTH - 1
def store_conv_checkpoints(x, conv_state, cu, offsets, rows, state_len) -> None:
KimiK3ROCmKDAPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(offsets, rows),
raw_qkv=x,
conv_state=conv_state,
cu_seqlens=cu,
)
def _conv_fixture(
seqlens: list[int], slots: int = 16, seed: int = 0, num_spec: int = 0
):
# kda_state_shape widens the conv row to conv_kernel_size - 1 + num_spec,
# but the window every reader starts from stays the leading STATE_LEN.
torch.manual_seed(seed)
total = sum(seqlens)
x = torch.randn(total, DIM, device="cuda", dtype=torch.bfloat16)
cu = torch.tensor(
[0] + torch.tensor(seqlens).cumsum(0).tolist(),
device="cuda",
dtype=torch.int32,
)
conv_state = torch.full(
(slots, DIM, STATE_LEN + num_spec),
float("nan"),
device="cuda",
dtype=torch.bfloat16,
)
return x, cu, conv_state
def test_conv_checkpoint_saves_window_before_offset() -> None:
seqlens = [200, 130]
x, cu, conv_state = _conv_fixture(seqlens)
offsets = _i32([192, 128])
rows = _i32([5, 9])
store_conv_checkpoints(x, conv_state, cu, offsets, rows, STATE_LEN)
for seq, (offset, row) in enumerate(zip([192, 128], [5, 9])):
end = int(cu[seq]) + offset
expected = x[end - STATE_LEN : end].transpose(0, 1)
torch.testing.assert_close(conv_state[row], expected)
def test_conv_checkpoint_skips_opted_out_sequences() -> None:
x, cu, conv_state = _conv_fixture([200, 200])
before = conv_state.clone()
store_conv_checkpoints(
x, conv_state, cu, _i32([0, 192]), _i32([5, NULL_BLOCK_ID]), STATE_LEN
)
assert torch.equal(conv_state.isnan(), before.isnan())
def test_exporter_stores_conv_window_through_shared_interface() -> None:
x, cu, conv_state = _conv_fixture([200, 200])
exporter = KimiK3ROCmKDAPrefillCheckpointExporter(state_len=STATE_LEN)
assert isinstance(exporter, MambaPrefillCheckpointExporter)
checkpoint = MambaPrefillCheckpointMetadata(
_i32([192, 0]), _i32([5, NULL_BLOCK_ID])
)
exporter.export(checkpoint, raw_qkv=x, conv_state=conv_state, cu_seqlens=cu)
torch.testing.assert_close(conv_state[5], x[192 - STATE_LEN : 192].transpose(0, 1))
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
untouched[5] = False
assert bool(conv_state[untouched].isnan().all())
@pytest.mark.parametrize("num_spec", [0, 2, 8])
def test_conv_checkpoint_matches_prefill_truncated_at_offset(num_spec: int) -> None:
# causal_conv1d_fn's own state write is the reference: a prefill that
# stopped at the offset leaves exactly the row a later prefix-cache hit
# reads back. Speculative decoding widens the row without moving that
# window, so a checkpoint sized off the row would sit num_spec tokens in
# the past.
seqlens = [200, 130]
offsets = [192, 64]
x, cu, conv_state = _conv_fixture(seqlens, seed=3, num_spec=num_spec)
weight = torch.randn(DIM, WIDTH, device="cuda", dtype=torch.bfloat16) * 0.3
bias = torch.randn(DIM, device="cuda", dtype=torch.bfloat16)
rows = [5, 9]
store_conv_checkpoints(x, conv_state, cu, _i32(offsets), _i32(rows), STATE_LEN)
# A prefill of just the prefix, into its own cache. Row 0 is the null block
# the conv kernel skips by design, so the reference rows start at 1.
prefix = torch.cat(
[x[int(cu[n]) : int(cu[n]) + off] for n, off in enumerate(offsets)]
)
prefix_cu = torch.tensor(
[0] + torch.tensor(offsets).cumsum(0).tolist(),
device="cuda",
dtype=torch.int32,
)
reference = torch.zeros_like(conv_state)
causal_conv1d_fn(
prefix.transpose(0, 1),
weight,
bias,
activation="silu",
conv_states=reference,
has_initial_state=torch.zeros(2, dtype=torch.bool, device="cuda"),
cache_indices=_i32([1, 2]),
query_start_loc=prefix_cu,
)
# causal_conv1d_fn writes only the leading STATE_LEN columns.
window = slice(None), slice(None, STATE_LEN)
torch.testing.assert_close(conv_state[rows[0]][window], reference[1][window])
torch.testing.assert_close(conv_state[rows[1]][window], reference[2][window])
def test_conv_checkpoint_leaves_spec_tail_of_row_alone() -> None:
# The draft-token columns belong to causal_conv1d_update. Filling the whole
# row would hand every reader, which indexes from column 0, a window
# shifted num_spec tokens into the past.
num_spec = 4
x, cu, conv_state = _conv_fixture([200], num_spec=num_spec, seed=11)
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([5]), STATE_LEN)
torch.testing.assert_close(
conv_state[5, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
)
assert bool(conv_state[5, :, STATE_LEN:].isnan().all()), (
"the checkpoint wrote into the speculative-decode tail of the row"
)
def test_conv_checkpoint_honours_transposed_cache_view() -> None:
# The SD conv layout reaches the layer transposed, not contiguous.
x, cu, _ = _conv_fixture([200])
raw = torch.full(
(16, STATE_LEN, DIM), float("nan"), device="cuda", dtype=torch.bfloat16
)
conv_state = raw.transpose(-1, -2)
assert not conv_state.is_contiguous()
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([7]), STATE_LEN)
torch.testing.assert_close(conv_state[7], x[192 - STATE_LEN : 192].transpose(0, 1))
untouched = torch.ones(16, dtype=torch.bool, device="cuda")
untouched[7] = False
assert bool(raw[untouched].isnan().all())
def test_checkpoint_metadata_indexes_prefill_rows_after_decodes() -> None:
# The chunk kernel receives the prefill tail alone, so the metadata has to
# be indexed the same way or it snapshots the wrong request's boundary.
md = _build(BatchSpec(seq_lens=[64, 200], query_lens=[1, 200]))
assert md.num_decodes == 1 and md.num_prefills == 1
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192]))
# Request 1 owns blocks 4..7, so its slot-2 state block is 6.
torch.testing.assert_close(md.checkpoint.state_indices, _i32([6]))
def test_conv_checkpoint_addresses_row_past_int32_limit() -> None:
# state_stride_0 is the whole mamba page, padded up to the attention page,
# so row * stride passes 2**31 a few thousand blocks into a large cache and
# an int32 wrap stores outside the tensor. The margin on the plain prefill
# path is 1.6x, which is luck: a bigger cache, a higher utilization or the
# wider row speculative decoding allocates all close it.
stride_row = DIM * (STATE_LEN + 8)
row = 2**31 // stride_row + 1
need = (row + 1) * stride_row * 2
free, _ = torch.accelerator.get_memory_info()
if free < need + (4 << 30):
pytest.skip(f"needs {need >> 30} GiB of free VRAM to reach the wrap")
x, cu, _ = _conv_fixture([200])
conv_state = torch.zeros(
row + 1, DIM, STATE_LEN + 8, device="cuda", dtype=torch.bfloat16
)
assert conv_state.stride(0) * row > 2**31, "fixture does not reach the wrap"
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([row]), STATE_LEN)
torch.testing.assert_close(
conv_state[row, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1)
)
assert not bool(conv_state[:row].any()), "the store wrote outside its row"
def test_conv_checkpoint_reads_row_strided_projection() -> None:
# mixed_qkv is a band of the fused QKVGFAB projection, so its row stride is
# the whole projection width, not DIM.
torch.manual_seed(5)
total, pad = 200, 177
projection = torch.randn(total, DIM + pad, device="cuda", dtype=torch.bfloat16)
x = projection[:, :DIM]
assert x.stride(0) == DIM + pad
cu = _i32([0, total])
conv_state = torch.full(
(16, DIM, STATE_LEN), float("nan"), device="cuda", dtype=torch.bfloat16
)
store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([4]), STATE_LEN)
torch.testing.assert_close(conv_state[4], x[192 - STATE_LEN : 192].transpose(0, 1))
def test_finer_prefix_match_unit_moves_checkpoint_to_hash_grid() -> None:
# A partial prefix hit matches at prefix_match_unit, so a state captured
# only at block boundaries is too coarse to resume from. Block 256 with
# unit 128 moves a 700-token prompt's checkpoint from 512 to 640.
md = _build(
BatchSpec(seq_lens=[700], query_lens=[700]),
block_size=256,
prefix_match_unit=128,
)
assert md.checkpoint is not None
torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([640]))
# Slot is still block-indexed: cdiv(700, 256) - 2 = 1, request 0 owns 0..2.
torch.testing.assert_close(md.checkpoint.state_indices, _i32([1]))
def test_checkpoint_metadata_skips_unaligned_offset() -> None:
# The scheduler declines a boundary off the published alignment and
# reserves nothing, so the builder has to reach the same verdict or it
# writes a block nobody hashed.
# Block 192, unit 96: a 700-token prompt's last hash boundary is 672, which
# is 32 tokens off the chunk grid.
md = _build(
BatchSpec(seq_lens=[700], query_lens=[700]),
block_size=192,
prefix_match_unit=96,
)
assert md.checkpoint is None
# ---------------------------------------------------------------------------
# Opting in
# ---------------------------------------------------------------------------
def _spec_for(monkeypatch, **attrs) -> MambaSpec:
"""Run the layer's `get_kv_cache_spec` over a stubbed base spec."""
base = MambaSpec(block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,))
monkeypatch.setattr(
GatedDeltaNetAttention, "get_kv_cache_spec", lambda self, cfg: base
)
layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention)
for name, value in dict(
use_prefill_checkpoint=True,
use_fused_chunk=True,
num_spec=0,
use_safe_gate=True,
).items():
setattr(layer, name, attrs.get(name, value))
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
vllm_config.cache_config.mamba_cache_mode = "align"
return layer.get_kv_cache_spec(vllm_config)
def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None:
# The scheduler learns the kernel's granularity only from the spec, so
# reserving a block without publishing the alignment would hash a state
# that is never written.
spec = _spec_for(monkeypatch)
assert spec.num_prefill_checkpoint_blocks == 1
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
@pytest.mark.parametrize(
"attr,value",
[
("use_prefill_checkpoint", False),
("use_fused_chunk", False),
("use_safe_gate", False),
],
)
def test_paths_that_cannot_export_stay_opted_out(monkeypatch, attr, value) -> None:
spec = _spec_for(monkeypatch, **{attr: value})
assert spec.num_prefill_checkpoint_blocks == 0
assert spec.prefill_checkpoint_alignment is None
def test_speculative_decoding_does_not_opt_out(monkeypatch) -> None:
# num_spec reaches the kernels as a wider conv row and an interleaved
# batch, both of which the layer handles.
spec = _spec_for(monkeypatch, num_spec=8)
assert spec.num_prefill_checkpoint_blocks == 1
assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT
+56 -6
View File
@@ -13,17 +13,17 @@ import torch
from vllm.platforms import current_platform
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
def _fused_chunk_available() -> bool:
if not (current_platform.is_rocm() and torch.cuda.is_available()):
return False
from vllm.platforms.rocm import on_gfx950
from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported
return on_gfx950()
return is_fused_kda_chunk_supported()
pytestmark = pytest.mark.skipif(
not _on_gfx950(),
reason="The fused KDA chunk kernel is only built for gfx950",
not _fused_chunk_available(),
reason="The fused KDA chunk kernel needs a gfx950 build that includes it",
)
HEAD_DIM = 128
@@ -701,6 +701,56 @@ def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
@pytest.mark.parametrize("use_fused", [False, True])
def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None:
# The layer passes block_table[:, 0], which is contiguous only while the
# table has one column. Speculative decoding reserves a state page per
# draft token, so the table widens and that column becomes strided.
_requires_kernel()
seqlens = [512, 320, 64]
inp = _inputs(seqlens, seed=43)
n = len(seqlens)
rows = [3 * i + 5 for i in range(n)]
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
ref_cache = cache.clone()
table = torch.zeros(n, 9, dtype=torch.int32, device="cuda")
table[:, 0] = idx
column = table[:, 0]
assert not column.is_contiguous()
o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused)
o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused)
torch.testing.assert_close(o_got, o_ref)
torch.testing.assert_close(cache, ref_cache)
@pytest.mark.parametrize("use_fused", [False, True])
def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None:
# A batch carrying speculative decodes pads its non-spec group with
# zero-length sequences whose cache row still belongs to a request, so
# consuming no tokens has to mean writing nothing and perturbing nobody.
_requires_kernel()
live = [200, 1, 128]
# _inputs draws h0 last, so the padded batch's live rows are the same draw
# as the unpadded one's and the two runs compare elementwise.
padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41)
rows = [3 * i + 5 for i in range(4)]
cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24)
before = cache.clone()
_run_paged(padded, cache, idx, warm, use_fused=use_fused)
ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24)
_run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused)
assert torch.equal(cache[rows[3]], before[rows[3]]), (
"the padded sequence's cache row was written"
)
torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()])
@pytest.mark.parametrize("groups", [2, 4])
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
"""At G>1 the read and the write are different blocks of one launch."""
+241
View File
@@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import (
)
from vllm.models.kimi_k3.nvidia import kda as nvidia_kda
from vllm.models.kimi_k3.nvidia.kda import (
KimiK3DeltaAttention,
_flashinfer_kda_prefill,
_flashkda_prefill,
is_flashinfer_fused_kda_decode_supported,
is_flashinfer_fused_kda_spec_decode_supported,
is_flashinfer_recurrent_kda_prefill_supported,
is_flashkda_supported,
is_fused_kda_decode_supported,
resolve_kda_spec_decode_backend,
)
from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata
from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM
from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
@@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch):
assert layer._forward(empty, empty, empty, empty, empty) is None
def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: True,
)
args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer"
assert resolve_kda_spec_decode_backend("native", *args, False) == "native"
assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer"
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported",
lambda *args: False,
)
assert resolve_kda_spec_decode_backend("auto", *args, False) == "native"
with pytest.raises(RuntimeError, match="packed_fused_kda_decode"):
resolve_kda_spec_decode_backend("flashinfer", *args, False)
def test_kda_recoverssm_config_state_layout():
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
@@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness(
torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2)
@torch.inference_mode()
@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_fused_kda_spec_decode_integration(
monkeypatch: pytest.MonkeyPatch,
state_dtype: torch.dtype,
):
if not is_flashinfer_fused_kda_spec_decode_supported(
num_heads=12,
head_dim=128,
conv_width=4,
num_spec=6,
input_dtype=torch.bfloat16,
conv_state_dtype=torch.bfloat16,
recurrent_state_dtype=state_dtype,
lower_bound=-5.0,
use_recoverssm=False,
):
pytest.skip("FlashInfer packed fused KDA decode is not supported")
generator = torch.Generator(device=DEVICE).manual_seed(20260820)
H, D, W, T = 12, 128, 4, 7
hidden_size = H * D
num_rows = 10
num_slots = 16
x = torch.randn(
num_rows,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
conv_weight = 0.1 * torch.randn(
3 * hidden_size,
W,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous()
raw_gate = torch.randn(
1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
raw_beta = torch.randn(
1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
output_gate = torch.randn(
num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator
)
A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE))
dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_(
-7.0, -4.0, generator=generator
)
norm_weight = torch.randn(
D, dtype=torch.float32, device=DEVICE, generator=generator
)
norm_eps = 1e-5
state_indices = torch.tensor(
[
list(range(1, T + 1)),
list(range(T + 1, 2 * T + 1)),
[0] * T,
],
dtype=torch.int32,
device=DEVICE,
)
query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE)
num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE)
conv_seed = (
0.5
* torch.randn(
num_slots,
T + W - 2,
3 * hidden_size,
dtype=torch.bfloat16,
device=DEVICE,
generator=generator,
)
).transpose(1, 2)
state_seed = (
0.5
* torch.randn(
num_slots,
H,
D,
D,
dtype=torch.float32,
device=DEVICE,
generator=generator,
)
).to(state_dtype)
metadata = KimiK3KDAMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=2,
num_spec_decode_tokens=num_rows,
num_actual_tokens=num_rows,
spec_query_start_loc=query_start_loc,
spec_state_indices_tensor=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
monkeypatch.setattr(
"vllm.models.kimi_k3.nvidia.kda.get_forward_context",
lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}),
)
def clone_strided(tensor: torch.Tensor) -> torch.Tensor:
clone = torch.empty_strided(
tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device
)
clone.copy_(tensor)
return clone
def composed_reference(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
) -> torch.Tensor:
mixed_qkv = causal_conv1d_update(
x,
conv_state,
conv_weight,
bias=None,
activation="silu",
conv_state_indices=state_indices[:, 0],
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=T,
validate_data=False,
out=torch.empty_like(x),
)
q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1)
recurrent_out, _ = fused_recurrent_kda_nvidia(
q=q.unsqueeze(0),
k=k.unsqueeze(0),
v=v.unsqueeze(0),
raw_g=raw_gate,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=dt_bias,
lower_bound=-5.0,
initial_state=recurrent_state,
cu_seqlens=query_start_loc,
ssm_state_indices=state_indices,
num_accepted_tokens=num_accepted_tokens,
)
recurrent_float = recurrent_out.float()
return (
recurrent_float
* torch.rsqrt(
recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps
)
* norm_weight
* output_gate.float().sigmoid().unsqueeze(0)
).to(torch.bfloat16)
def run_flashinfer(
conv_state: torch.Tensor,
recurrent_state: torch.Tensor,
output: torch.Tensor,
) -> torch.Tensor:
layer = SimpleNamespace(
prefix="test.layer",
kv_cache=(conv_state.transpose(-1, -2), recurrent_state),
kda_spec_decode_backend="flashinfer",
decode_conv1d_weight=fused_weight,
decode_norm_weight=norm_weight,
gate_lower_bound=-5.0,
A_log=A_log,
dt_bias=dt_bias,
o_norm=SimpleNamespace(eps=norm_eps),
)
KimiK3DeltaAttention._forward(
layer,
mixed_qkv=x,
g1=raw_gate,
g2=output_gate,
beta=raw_beta,
core_attn_out=output,
)
return output
expected_conv = clone_strided(conv_seed)
expected_state = state_seed.clone()
expected = composed_reference(expected_conv, expected_state)
actual_conv = clone_strided(conv_seed)
actual_state = state_seed.clone()
output = torch.empty_like(expected)
actual = run_flashinfer(actual_conv, actual_state, output)
assert actual.data_ptr() == output.data_ptr()
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2)
assert torch.count_nonzero(actual[:, :7]) > 0
torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured = run_flashinfer(actual_conv, actual_state, output)
actual_conv.copy_(conv_seed)
actual_state.copy_(state_seed)
query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE))
num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE))
expected_conv.copy_(conv_seed)
expected_state.copy_(state_seed)
expected = composed_reference(expected_conv, expected_state)
graph.replay()
torch.accelerator.synchronize()
assert captured.data_ptr() == output.data_ptr()
torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2)
torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0)
torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2)
def test_fused_kda_decode_rejects_speculative_conv_state():
assert not is_fused_kda_decode_supported(
num_heads=12,
@@ -259,7 +259,7 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
model_output = outputs[0].outputs[0].text.strip()
assert model_output.startswith(tool_parser.bot_token), model_output
parsed_message = tool_parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
parsed_message = tool_parser.extract_tool_calls(model_output, None)
assert parsed_message.tools_called
@@ -302,7 +302,7 @@ def test_mistral_function_call_nested_json():
model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}"
parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
parsed = parser.extract_tool_calls(model_output, None)
# Assertions: the tool call is detected and the full nested JSON is parsed
# without truncation.
@@ -335,7 +335,7 @@ def test_mistral_function_call_nested_json():
]
)
parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type]
parsed = parser.extract_tool_calls(model_output, None)
# Assertions: the tool call is detected and the full nested JSON is parsed
# without truncation.
@@ -7,6 +7,15 @@ import pytest
from tests.conftest import HfRunner
from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
def float_embeddings(response: EmbeddingResponse) -> list[list[float]]:
embeddings = []
for d in response.data:
assert isinstance(d.embedding, list)
embeddings.append(d.embedding)
return embeddings
def run_embedding_correctness_test(
@@ -104,7 +104,16 @@ def run_test(
)
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
@pytest.mark.parametrize(
"model",
[
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
# instead of shipping a separate lm_head.weight tensor; this
# exercises that path (see tie_weights() in nemotron_parse.py).
"nvidia/NVIDIA-Nemotron-Parse-2.0",
],
)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
@@ -1,198 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from importlib.metadata import version
import pytest
import regex as re
from packaging.version import Version
from transformers import AutoModelForCausalLM, AutoTokenizer
from vllm.logprobs import SampleLogprobs
from vllm.multimodal.image import rescale_image_size
from ....conftest import (
IMAGE_ASSETS,
HfRunner,
PromptImageInput,
VllmRunner,
)
from ....utils import multi_gpu_test
from ...utils import check_logprobs_close
pytestmark = pytest.mark.skipif(
Version("5.0") <= Version(version("transformers")),
reason=(
"vllm upgraded transformers above v5.4 where HF model custom code uses siglip2 "
"internals (filter_out_non_signature_kwargs) removed by "
"huggingface/transformers#43514"
),
)
MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B"
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|user|>\n<image>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
"cherry_blossom": "<|user|>\n<image>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
}
)
HF_MULTIIMAGE_IMAGE_PROMPT = (
"<|user|>\n<image>\n<image>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
)
DTYPE = "half"
MAX_TOKENS = 128
NUM_LOGPROBS = 10
def vllm_to_hf_output(
vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str
):
"""Sanitize vllm output to be comparable with hf output."""
_, output_str, out_logprobs = vllm_output
output_str_without_image = re.sub(r"(<image>)+", "", output_str)
if output_str_without_image and output_str_without_image[0] == " ":
output_str_without_image = output_str_without_image[1:]
hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
hf_output_ids = tokenizer.encode(output_str_without_image)
if hf_output_ids and hf_output_ids[0] == tokenizer.bos_token_id:
hf_output_ids = hf_output_ids[1:]
return hf_output_ids, hf_output_str, out_logprobs
def _build_single_image_inputs(
image_assets,
) -> list[tuple[list[str], PromptImageInput]]:
"""Build single-image inputs for all size_factors at once."""
images = [asset.pil_image for asset in image_assets]
all_inputs: list[tuple[list[str], PromptImageInput]] = []
for size_factors in [[1.0], [0.25, 0.5, 1.0]]:
for image, prompt in zip(images, HF_IMAGE_PROMPTS):
all_inputs.append(
(
[prompt for _ in size_factors],
[rescale_image_size(image, f) for f in size_factors],
)
)
return all_inputs
def _build_multi_image_inputs(
image_assets,
) -> list[tuple[list[str], PromptImageInput]]:
"""Build multi-image inputs for all size_factors at once."""
images = [asset.pil_image for asset in image_assets]
all_inputs: list[tuple[list[str], PromptImageInput]] = []
for size_factors in [[0.5], [0.15, 0.30]]:
all_inputs.append(
(
[HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
[
[rescale_image_size(image, factor) for image in images]
for factor in size_factors
],
)
)
return all_inputs
def _run_and_compare(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
all_inputs: Sequence[tuple[list[str], PromptImageInput]],
model: str,
max_model_len: int,
max_num_seqs: int,
mm_limit: int,
gpu_memory_utilization: float,
):
"""Load each runner once, run all inputs, then compare."""
# NOTE: run vLLM first, then HF. vLLM needs a fresh process without
# cuda initialization; running HF first would break the multiprocessing
# backend with fork method.
with vllm_runner(
model,
runner="generate",
max_model_len=max_model_len,
max_num_seqs=max_num_seqs,
gpu_memory_utilization=gpu_memory_utilization,
dtype=DTYPE,
limit_mm_per_prompt={"image": mm_limit},
tensor_parallel_size=2,
trust_remote_code=True,
enforce_eager=True,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.generate_greedy_logprobs(
prompts,
MAX_TOKENS,
num_logprobs=NUM_LOGPROBS,
images=images,
)
for prompts, images in all_inputs
]
hf_model_kwargs = {"_attn_implementation": "sdpa", "device_map": "auto"}
with hf_runner(
model,
dtype=DTYPE,
model_kwargs=hf_model_kwargs,
auto_cls=AutoModelForCausalLM,
trust_remote_code=True,
) as hf_model:
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
MAX_TOKENS,
num_logprobs=NUM_LOGPROBS,
images=images,
)
for prompts, images in all_inputs
]
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("model", [MODEL_ID])
def test_models(hf_runner, vllm_runner, image_assets, model) -> None:
all_inputs = _build_single_image_inputs(image_assets)
_run_and_compare(
hf_runner,
vllm_runner,
all_inputs,
model,
max_model_len=8192,
max_num_seqs=2,
mm_limit=1,
gpu_memory_utilization=0.80,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("model", [MODEL_ID])
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model) -> None:
all_inputs = _build_multi_image_inputs(image_assets)
_run_and_compare(
hf_runner,
vllm_runner,
all_inputs,
model,
max_model_len=8192,
max_num_seqs=2,
mm_limit=2,
gpu_memory_utilization=0.80,
)
@@ -5,6 +5,7 @@ from collections.abc import Sequence
from typing import Any
import pytest
import torch
from transformers import AutoModelForSpeechSeq2Seq
from vllm.assets.audio import AudioAsset
@@ -264,6 +265,49 @@ def test_parse_language_detection_output():
cls.parse_language_detection_output([], make_tokenizer("anything"))
def test_create_fake_bias_for_k_proj_injects_out_features():
"""Packed k_proj weights get a zeros bias of length ``out_features``."""
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
packed = torch.ones(2, 16)
result = dict(
_create_fake_bias_for_k_proj(
[("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)],
".k_proj.weight_packed",
out_features=8,
)
)
bias_name = "model.encoder.layers.0.self_attn.k_proj.bias"
assert bias_name in result
assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result
assert tuple(result[bias_name].shape) == (8,)
assert torch.equal(result[bias_name], torch.zeros(8))
def test_create_fake_bias_for_k_proj_forwards_real_bias():
"""A checkpoint bias is forwarded and no extra fake bias is emitted."""
from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj
weight = torch.ones(8, 16)
real_bias = torch.ones(8)
items = list(
_create_fake_bias_for_k_proj(
[
("layers.0.self_attn.k_proj.weight", weight),
("layers.0.self_attn.k_proj.bias", real_bias),
],
".k_proj.weight",
out_features=8,
)
)
names = [n for n, _ in items]
assert names == [
"layers.0.self_attn.k_proj.weight",
"layers.0.self_attn.k_proj.bias",
]
assert items[1][1] is real_bias
@pytest.mark.core_model
@pytest.mark.cpu_model
@pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"])
@@ -1193,7 +1193,7 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
output.sequences = output.sequences[:, prompt_len:]
return output
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment]
hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign]
hf_model.model.generate = patched_generate # type: ignore[method-assign]
return hf_model
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import GenerateModelInfo
from .ppl_utils import vqa_ppl_test
MODELS = [
(GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28),
]
def _pixel_size(factor: int):
# Match max image token size with qwen's PPL test
return {
"size": {
"shortest_edge": 2 * factor**2,
"longest_edge": 2 * 1280 * factor**2,
}
}
@pytest.mark.parametrize("model_info,factor", MODELS)
@pytest.mark.parametrize("mm_device_do_normalize", [True, False])
def test_ppl(
hf_runner,
vllm_runner,
model_info: GenerateModelInfo,
factor: int,
mm_device_do_normalize: bool,
):
vqa_ppl_test(
hf_runner,
vllm_runner,
model_info,
vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize},
mm_processor_kwargs=_pixel_size(factor),
)
+1 -1
View File
@@ -132,7 +132,7 @@ def test_models(
_run_test(
hf_runner,
vllm_runner,
input_cases, # type: ignore[arg-type]
input_cases,
model,
dtype=dtype,
)
@@ -23,7 +23,13 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionContentPartTextParam,
)
from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.nemotron_vl import (
SIGLIP_MEAN,
SIGLIP_STD,
LlamaNemotronVLEmbedImageProcessor,
)
from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
from ....utils import ROCM_ENGINE_KWARGS
@@ -52,6 +58,54 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
MODELS = ["nvidia/llama-nemotron-embed-vl-1b-v2"]
@pytest.mark.usefixtures("default_vllm_config")
def test_device_normalization_matches_cpu_transform() -> None:
device = current_platform.device_type
image = Image.new("RGB", (48, 32), color=(12, 128, 240))
processor_kwargs = dict(
image_size=32,
min_dynamic_patch=1,
max_dynamic_patch=1,
dynamic_image_size=True,
use_thumbnail=False,
)
cpu_processor = LlamaNemotronVLEmbedImageProcessor(**processor_kwargs)
device_processor = LlamaNemotronVLEmbedImageProcessor(
**processor_kwargs, do_rescale=False, do_normalize=False
)
expected = cpu_processor._images_to_pixel_values_lst([image])[0]
raw_pixels = device_processor._images_to_pixel_values_lst([image])[0]
input_norm = FusedMMInputNorm(
image_mean=list(SIGLIP_MEAN),
image_std=list(SIGLIP_STD),
rescale_factor=1 / 255,
).to(device)
actual = input_norm(
raw_pixels.flatten(start_dim=1).to(device), expected.dtype
).view_as(expected)
assert raw_pixels.dtype == torch.uint8
torch.testing.assert_close(actual, expected.to(device))
@pytest.mark.parametrize(("do_rescale", "do_normalize"), [(True, False), (False, True)])
def test_device_normalization_rejects_partial_cpu_processing(
do_rescale: bool, do_normalize: bool
) -> None:
with pytest.raises(ValueError, match="both enabled or both disabled"):
LlamaNemotronVLEmbedImageProcessor(
image_size=32,
min_dynamic_patch=1,
max_dynamic_patch=1,
dynamic_image_size=True,
use_thumbnail=False,
do_rescale=do_rescale,
do_normalize=do_normalize,
)
def _run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
@@ -136,7 +190,7 @@ def test_models(
_run_test(
hf_runner,
vllm_runner,
input_cases, # type: ignore[arg-type]
input_cases,
model,
dtype=dtype,
)
@@ -149,7 +149,7 @@ def test_models(
_run_test(
hf_runner,
vllm_runner,
input_cases, # type: ignore[arg-type]
input_cases,
model,
dtype=dtype,
)
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek-V4 VL processing helpers."""
from PIL import Image
from vllm.models.deepseek_v4.common.mm_preprocess import (
DeepseekV4VLProcessingInfo,
DeepseekV4VLProcessor,
build_image_block_pad_free,
)
from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config
class _ConfigCtx:
def __init__(self, config: DeepseekV4Config) -> None:
self._config = config
def get_hf_config(self, *args, **kwargs):
return self._config
def _feature_counts(
processor: DeepseekV4VLProcessor, width: int, height: int
) -> tuple[int, int]:
out = processor(images=[Image.new("RGB", (width, height))])
n_llm_h, n_llm_w = out["llm_grid"][0].tolist()
types, _ = build_image_block_pad_free(n_llm_h, n_llm_w)
return len(types), out["patches"].shape[0]
def test_get_image_size_with_most_features_beats_wide_real_image():
"""Dummy size must maximize tokens/patches within the budget (#59271).
A square under-counts because each aligner row pays an IMAGE_NEW_LINE.
"""
config = DeepseekV4Config(vision_n_layers=24)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
processor = DeepseekV4VLProcessor(config)
dummy = info.get_image_size_with_most_features()
dummy_features = _feature_counts(processor, dummy.width, dummy.height)
wide_features = _feature_counts(processor, 1932, 336)
assert dummy == (1932, 336)
assert wide_features <= dummy_features
def test_get_image_size_with_most_features_respects_max_wh_ratio():
config = DeepseekV4Config(vision_n_layers=24)
info = DeepseekV4VLProcessingInfo(_ConfigCtx(config))
dummy = info.get_image_size_with_most_features()
assert dummy.width / dummy.height <= config.vision_max_wh_ratio
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Video placeholder accounting for GLM-5.3-Flash.
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
placeholders per timestamp, so the timestamps returned by
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
while ``video_grid_thw`` decides how many rows the vision tower produces. If
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
inside a worker and takes the engine down with it.
The checks below are arithmetic: no weights, no GPU.
"""
import pytest
import torch
from PIL import Image
from transformers.models.glm5_next.video_processing_glm5_next import (
Glm5NextVideoProcessor,
smart_resize,
)
from transformers.video_utils import VideoMetadata
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
@pytest.fixture(scope="module")
def processor():
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"video": 1},
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
def _pixel_path_grid(
video_processor: Glm5NextVideoProcessor,
num_frames: int,
height: int,
width: int,
) -> tuple[int, int, int]:
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
resized_height, resized_width = smart_resize(
num_frames=num_frames,
height=height,
width=width,
temporal_factor=video_processor.temporal_patch_size,
factor=video_processor.patch_size
* video_processor.merge_size
* video_processor.patch_expand_factor,
min_pixels=video_processor.min_image_tokens,
max_pixels=video_processor.max_image_tokens,
)
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
return (
padded_frames // video_processor.temporal_patch_size,
resized_height // video_processor.patch_size,
resized_width // video_processor.patch_size,
)
@pytest.mark.parametrize(
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
[
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
# 1080p, 20 s: same factor of 3 at a full-size canvas.
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
# 1080p, 60 s: the one duration window where the two samplers agree
# anyway -- a regression guard, the count must not move.
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
],
)
def test_video_placeholders_match_encoder_rows(
processor,
total_num_frames: int,
fps: float,
duration: float,
height: int,
width: int,
expected_grid: tuple[int, int, int],
):
info = processor.info
video_processor = info.get_video_processor()
frame_indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
)
grid_t, grid_h, grid_w = _pixel_path_grid(
video_processor, len(frame_indices), height, width
)
assert (grid_t, grid_h, grid_w) == expected_grid
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": total_num_frames,
"fps": fps,
"duration": duration,
"do_sample_frames": True,
},
total_num_frames,
)
merge_length = video_processor.merge_size**2
tokens_per_frame = grid_h * grid_w // merge_length
encoder_rows = grid_t * grid_h * grid_w // merge_length
assert len(timestamps) == grid_t
assert len(timestamps) * tokens_per_frame == encoder_rows
assert timestamps == sorted(timestamps)
assert timestamps[0] == 0
assert timestamps[-1] <= duration
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
"""The loader may pre-sample and hand the frames over as they are."""
info = processor.info
video_processor = info.get_video_processor()
num_frames = 32
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": 256,
"fps": 8.0,
"duration": 32.0,
"do_sample_frames": False,
"frames_indices": list(range(0, 256, 256 // num_frames)),
},
num_frames,
)
assert len(timestamps) == grid_t
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
"""A clip the sampler cannot pick a single frame from is a bad request."""
with pytest.raises(ValueError, match="selected no frames"):
processor.info._get_video_second_idx_glm46v(
{
"total_num_frames": 2,
"fps": 8.0,
"duration": 0.25,
"do_sample_frames": True,
},
2,
)
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 2},
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
@@ -4,14 +4,29 @@
import pytest
import torch
import torch.nn.functional as F
from PIL import Image
from transformers import AutoProcessor, BatchFeature
from transformers.models.pixtral import PixtralProcessor
from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.models.lightonocr import (
LightOnOCRForConditionalGeneration,
LightOnOCRProcessingInfo,
)
from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo
from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo
from vllm.model_executor.models.pixtral import (
PixtralHFEncoderInfo,
pixtral_patch_embed,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalKwargsItems
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from ...utils import build_model_context
@@ -21,6 +36,65 @@ _MODEL_CONFIG_KWARGS = {"config_format": "hf"}
_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
_LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025"
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.skipif(
not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("normalize", [False, True])
@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))])
def test_pixtral_patch_embed_preserves_image_order_and_strides(
dtype: torch.dtype,
normalize: bool,
channels: int,
patch_shape: tuple[int, int],
):
"""Packed projection matches CHW normalization and patch convolution."""
device = torch.device(current_platform.device_type)
norm = (
FusedMMInputNorm(
_RGB_MEAN[:channels],
_RGB_STD[:channels],
_RGB_RESCALE,
channel=channels,
).to(device)
if normalize
else IdentityInputNorm()
)
weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype)
images = [
torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8),
torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[
:, 1:17, 1:17
],
torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[
:, 1:17, 2:18
],
]
reference = []
for image in images:
if normalize:
normalized = (
image.float() * norm.weight[:, None, None] + norm.bias[:, None, None]
)
else:
normalized = image
reference.append(
F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape)
)
packed, actual = pixtral_patch_embed(images, weight, norm)
for expected, result in zip(reference, actual):
torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02)
expected_packed = torch.cat(
[result.flatten(2).transpose(1, 2) for result in reference], dim=1
)
torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02)
def _process_images_with_hf(
hf_processor,
@@ -128,6 +202,9 @@ def test_processor_size_override(
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
assert isinstance(vllm_hf_processor, PixtralProcessor)
hf_processor = AutoProcessor.from_pretrained(
model_id,
fix_mistral_regex=True,
@@ -153,6 +230,53 @@ def test_processor_size_override(
assert prompt_update_tokens == hf_placeholder_tokens
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 2},
model_config_kwargs=_MODEL_CONFIG_KWARGS,
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (31, 47), color=(17, 89, 231)),
Image.new("RGB", (48, 32), color=(201, 13, 127)),
]
prompt = [processor.info.get_hf_config().image_token_index] * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert all(value.dtype == torch.uint8 for value in raw_values)
input_norm = build_mm_input_norm(ctx.model_config)
patch_size = processor.info.get_hf_config().vision_config.patch_size
def pack_patches(image: torch.Tensor) -> torch.Tensor:
channels, height, width = image.shape
rows = height // patch_size
cols = width // patch_size
return (
image[:, : rows * patch_size, : cols * patch_size]
.reshape(channels, rows, patch_size, cols, patch_size)
.permute(1, 3, 0, 2, 4)
.reshape(-1, channels * patch_size**2)
)
for raw, normalized in zip(raw_values, normalized_values):
output = input_norm(pack_patches(raw), normalized.dtype)
torch.testing.assert_close(
output, pack_patches(normalized), rtol=1e-5, atol=1e-6
)
def test_scoped_request_size_overrides_configured_flat_size():
ctx = build_model_context(
_MODEL_ID,
@@ -188,6 +312,8 @@ def test_scoped_request_size_overrides_configured_flat_size():
def test_lightonocr_keeps_vision_config_image_size():
assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize
ctx = build_model_context(
_LIGHTON_MODEL_ID,
mm_processor_kwargs={"size": {"longest_edge": 1008}},
@@ -106,7 +106,7 @@ def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedenc
mm_device_do_normalize=False,
)
model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config)
ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type]
ctx = InputProcessingContext(model_config, tokenizer=None)
info = Qwen2VLProcessingInfo(ctx)
monkeypatch.setattr(
info,
@@ -236,7 +236,7 @@ def test_get_vision_info_reads_merged_modality_scope(
do_resize=False,
image_processor=image_processor,
mm_kwargs={},
modality=modality, # type: ignore[arg-type]
modality=modality,
)
assert seen == [merged[scope]]
@@ -0,0 +1,176 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
device gate accepts the card, so the checkpoint could not be served natively at
all.
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
a correction bias), so the backend only has to ask for it. This module covers
both halves: the gate accepts the router family, and the routing the wrapper
requests reproduces the model's own math.
"""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
MIMO_TOP_K = 8
def mimo_routing_method() -> RoutingMethodType:
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
return get_routing_method_type(
scoring_func="sigmoid",
top_k=MIMO_TOP_K,
renormalize=True,
num_expert_group=1,
has_e_score_bias=True,
)
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=MIMO_TOP_K,
hidden_dim=4096,
intermediate_size=2048,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device=torch.device("cuda"),
routing_method=mimo_routing_method(),
router_logits_dtype=torch.float32,
)
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
def test_aiter_w4a16_supports_mimo_routing() -> None:
"""The gate accepts MiMo's router family — the reported construction abort.
Only the predicate the fix touches is exercised, so this runs on any host;
the full config path is covered by the test below.
"""
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
)
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
"""The same gate through the real config path, device check included."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
AiterW4A16ExpertsMonolithic,
mimo_moe_config(),
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
"""The routing the wrapper requests must equal MiMo's own router math.
MiMo's router is ungrouped: sigmoid scores, selection on
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
"""
aiter_routing = pytest.importorskip(
"aiter.ops.triton.moe.moe_routing.routing"
).routing
torch.manual_seed(0)
top_k, num_tokens = MIMO_TOP_K, 16
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
bias = torch.randn(num_experts, device="cuda") * 0.5
# Reference: select on the biased score, return the unbiased score, renorm.
scores = torch.sigmoid(logits.float())
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
ref_weights = scores.gather(1, ref_ids)
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
routing_data, topk_indx, _ = aiter_routing(
logits,
top_k,
score_mode=score_mode,
bias=bias,
renorm=True,
routed_scaling_factor=1.0,
use_grouped_topk=False,
)
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
# i-th entry of the expert-sorted array whose weights are gate_scal.
hist = routing_data.expt_hist.to(torch.long).cpu()
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
slot = topk_indx.reshape(-1).to(torch.long).cpu()
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
out = torch.zeros(num_tokens, top_k)
for i in range(slot.numel()):
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
ids[token, position] = int(expert_of_slot[i])
out[token, position] = float(weights[i])
return ids, out
got_ids, got_weights = routed("sigmoid")
# aiter's triton top-k emits each token's selection sorted by *ascending
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
# rank order — align both sides on the expert id so the weight check
# becomes elementwise and still validates the expert<->weight pairing.
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
order = torch.argsort(ids, dim=-1)
return ids.gather(1, order), w.gather(1, order)
got_ids, got_weights = align(got_ids, got_weights)
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
assert torch.equal(got_ids, ref_ids)
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
# Control: the correction bias must steer selection only. Ignoring it would
# pick from the plain sigmoid scores instead.
plain_ids = scores.topk(top_k, dim=-1).indices
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
+5 -19
View File
@@ -10,6 +10,7 @@ from unittest.mock import patch
import pytest
import torch
from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig
from vllm.config.speculative import SpeculativeConfig
from vllm.model_executor.models.config import (
@@ -17,10 +18,6 @@ from vllm.model_executor.models.config import (
Qwen4ExpForConditionalGenerationConfig,
)
from vllm.models.qwen4_exp.common.mtp import make_mtp_hidden_buffer
from vllm.models.qwen4_exp.config import (
Qwen4ExpConfig,
Qwen4ExpTextConfig,
)
from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
@@ -42,7 +39,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig:
"linear_num_value_heads": 2,
"linear_key_head_dim": 8,
"linear_value_head_dim": 8,
"num_experts": 0,
"num_experts": 4,
"num_experts_per_tok": 2,
# `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled.
"eos_token_id": 1,
"hc_count": 2,
"hc_lowrank": 4,
"ple_layer_ids": [1],
@@ -287,20 +287,6 @@ def test_qwen4_exp_model_state_prepares_stable_dummy_ngram_inputs() -> None:
assert second["ngram_context"].data_ptr() == ngram_context_ptr
def test_text_config_normalizes_transformers_sparse_attention_spelling():
"""Checkpoints re-exported through transformers' Qwen4Exp config class
serialize "qwen_sparse_attention"; vLLM keys QSA off "full_attention"."""
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
config = Qwen4ExpTextConfig(
hc_count=4,
layer_types=["linear_attention", "qwen_sparse_attention"],
num_hidden_layers=2,
)
assert config.layer_types == ["linear_attention", "full_attention"]
@contextmanager
def _captured_transformers_warnings() -> Iterator[list[str]]:
"""Records what transformers logs, without relying on propagation.
+107 -1
View File
@@ -15,6 +15,7 @@ from safetensors import safe_open
from safetensors.torch import load, load_file, save_file
from torch import nn
from torch.nn import functional as F
from transformers import Qwen4ExpTextConfig
import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module
import vllm.model_executor.parameter as parameter_module
@@ -39,7 +40,6 @@ from vllm.models.qwen4_exp.common.ple import (
compute_ple_shard_overlap,
copy_ple_embedding_shard_,
)
from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig
from vllm.models.qwen4_exp.nvidia.ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEDeviceEmbedding,
@@ -616,6 +616,32 @@ def test_pinned_embedding_forward_finalizes_prefetched_output(
assert torch.equal(output, expected)
def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None:
"""Forward before any start_prefetch must fail loudly, not read garbage."""
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = None
embedding._prefetch_stream = None
embedding._output_dim = 6
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="prior start_prefetch"):
embedding(hidden_states)
def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None:
"""_finalize_prefetch before any start_prefetch must fail loudly."""
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
embedding._prefetch_buffer = None
embedding._prefetch_stream = None
embedding.tp_size = 1
output = torch.zeros(2, 6, dtype=torch.bfloat16)
with pytest.raises(RuntimeError, match="prior start_prefetch"):
embedding._finalize_prefetch(torch.empty(4, 2, 3), output)
def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None:
embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding)
nn.Module.__init__(embedding)
@@ -730,6 +756,67 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva(
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""start_prefetch lazily allocates the stream/buffer, then forward consumes it.
Exercises the full pinned-prefetch contract end to end: first call
allocates the side stream and buffer with the constructor geometry,
and forward finalizes the lookup into the returned output.
"""
_mock_etp_group(monkeypatch)
monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
embedding_module, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
parameter_module, "get_tensor_model_parallel_world_size", lambda: 1
)
with torch.device("cuda:0"):
embedding = Qwen4ExpPLEPinnedHostEmbedding(
4,
3,
params_dtype=torch.bfloat16,
padding_size=1,
prefix="test.ple_embedding",
embedding_method=Qwen4ExpPLEUnquantizedEmbeddingMethod(),
num_ngram_heads=2,
max_total_tokens=4,
)
assert embedding._prefetch_buffer is None
assert embedding._prefetch_stream is None
loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3)
loaded_weight = loaded_weight.to(torch.bfloat16)
copy_ple_embedding_shard_(
embedding.weight,
loaded_weight,
checkpoint_start=0,
tp_start=0,
tp_end=4,
)
ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0")
hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0")
embedding.start_prefetch(hidden_states, ngram_ids)
buffer = embedding._prefetch_buffer
assert buffer is not None
assert buffer.shape == (4, 2, 3)
assert buffer.dtype == torch.bfloat16
assert buffer.device.type == "cuda"
assert embedding._prefetch_stream is not None
output = embedding(hidden_states)
expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def test_ple_fp8_embedding_supports_mixed_precision_config() -> None:
prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
quant_config = ModelOptMixedPrecisionConfig.from_config(
@@ -2355,6 +2442,25 @@ def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale(
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
def test_amd_pinned_embedding_prefetch_machinery_is_lazy(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The pinned embedding must not allocate the prefetch stream/buffer in init.
Paths that never prefetch (AMD uses the synchronous pinned-lookup op)
keep the stream/buffer unallocated for the process lifetime. NVIDIA's
first ``start_prefetch`` — the eager profile run, always before cudagraph
capture — allocates them lazily.
"""
module, _ = _build_amd_ngram_embedding(
monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False
)
embedding = module.ngram_embedding
assert embedding._prefetch_buffer is None
assert embedding._prefetch_stream is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm")
@pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"])
def test_amd_pinned_embedding_output_written_under_compile(
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA pre-indexer."""
"""Correctness tests for the fused QSA prepare kernel."""
import pytest
import torch
@@ -15,9 +15,7 @@ from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import (
qsa_pre_indexer,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
@@ -36,6 +34,7 @@ ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
def _make_block_table(block_counts):
@@ -61,6 +60,84 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
"""Random main-attention arguments with a paged cache in vLLM's layout."""
num_blocks = num_tokens // MAIN_PAGE + 2
cache = torch.zeros(
num_blocks,
MAIN_HK,
MAIN_PAGE,
2 * MAIN_D,
dtype=torch.uint8 if fp8_cache else torch.bfloat16,
device="cuda",
).transpose(1, 2)
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
slots[-1] = -1
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
return dict(
main_qkv=torch.randn(
num_tokens,
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
dtype=torch.bfloat16,
device="cuda",
),
main_q_norm_weight=norm_weights[0],
main_k_norm_weight=norm_weights[1],
main_eps=EPS,
main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache,
main_slot_mapping=slots,
main_k_scale=0.5,
main_v_scale=2.0,
)
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
"""Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash."""
from vllm._custom_ops import reshape_and_cache_flash
from vllm.model_executor.layers.fused_qk_norm_rope import (
fused_qk_rmsnorm_rope_gate,
)
q_gate, k, v = main["main_qkv"].split(
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
)
q, k, gate = fused_qk_rmsnorm_rope_gate(
q_gate,
k,
main["main_q_norm_weight"],
main["main_k_norm_weight"],
rope.cos_sin_cache,
positions,
EPS,
MAIN_HQ,
MAIN_HK,
MAIN_D,
rope.rotary_dim,
mrope_section=MROPE_SECTION if positions.ndim == 2 else None,
norm_beta=1.0,
)
kv_cache = main["main_kv_cache"]
fp8_cache = kv_cache.dtype == torch.float8_e4m3fn
cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache)
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
reshape_and_cache_flash(
k.view(-1, MAIN_HK, MAIN_D),
v.view(-1, MAIN_HK, MAIN_D),
key_cache,
value_cache,
main["main_slot_mapping"],
"fp8" if fp8_cache else "auto",
torch.tensor(main["main_k_scale"], device="cuda"),
torch.tensor(main["main_v_scale"], device="cuda"),
)
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
assert torch.equal(gate_out.flatten(1), gate)
if fp8_cache:
assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn))
else:
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@@ -86,7 +163,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_pre_indexer_matches_unfused(
def test_qsa_fused_prepare_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
@@ -246,7 +323,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
qsa_pre_indexer(
# The main-attention cache follows the recipe's pairing with the indexer.
main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn)
main_q_out, main_gate_out = qsa_prepare(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
@@ -266,7 +345,9 @@ def test_qsa_fused_pre_indexer_matches_unfused(
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
**main,
)
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
+15 -4
View File
@@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
rope_position_offset=0,
),
compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)),
use_fused_pre_indexer=True,
rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)),
q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_layernorm=SimpleNamespace(weight=torch.ones(1)),
compress_ratio=2,
)
attn = SimpleNamespace(
use_fused_qsa_prepare=True,
kv_cache=torch.empty(0, 1, 1, 2),
kv_cache_dtype="auto",
q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6),
k_norm=SimpleNamespace(weight=torch.ones(1)),
_k_scale_float=1.0,
_v_scale_float=1.0,
)
monkeypatch.setattr(
indexer_qsa,
"qsa_pre_indexer",
"qsa_prepare",
lambda *args, **kwargs: updates.append((args, kwargs)),
)
monkeypatch.setattr(
@@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection(
lambda *args, **kwargs: selections.append((args, kwargs)),
)
actual = indexer_qsa.QSAIndexer.forward(
actual, _ = indexer_qsa.QSAIndexer.forward(
indexer,
torch.zeros(2, 2),
torch.tensor([7, 8]),
rows,
attn=attn,
qkv=torch.zeros(2, 4),
slot_mapping=torch.arange(2),
)
assert actual is rows
@@ -573,7 +584,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
_metadata=lambda: (raw_metadata, compressed_metadata),
skip_topk=True,
index_kv_heads=1,
use_fused_pre_indexer=False,
index_n_heads=1,
index_head_dim=64,
indexer_dtype=torch.bfloat16,
@@ -596,6 +606,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None:
torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1),
torch.zeros(8, dtype=torch.long, device=device),
torch.full((5, 5), -1, dtype=torch.int32, device=device),
attn=SimpleNamespace(use_fused_qsa_prepare=False),
)
torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]])
expected_compressed = torch.zeros_like(compressed_cache)
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""
import pytest
import torch
from vllm.forward_context import (
ForwardContext,
get_forward_context,
override_forward_context,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
DEVICE = torch.device("cuda")
NUM_TOKENS = 401
HC, HIDDEN = 2, 3
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
WINDOW = 128
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
def _context(metadata):
return ForwardContext(
no_compile_layers={},
attn_metadata=metadata,
slot_mapping={},
is_padding=torch.zeros(NUM_TOKENS, dtype=torch.bool, device=DEVICE),
)
def _set_replay_batch(layers, rows, metadata):
layers.rows = torch.tensor(rows, device=DEVICE)
layers.forward_context = ForwardContext(
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
)
def _states(num_tokens: int):
"""(hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual),
shaped as the model's: hidden states [T, H], mixes [T, HC], residual [T, HC, H]."""
hidden = torch.arange(num_tokens, device=DEVICE, dtype=torch.float32)
hidden = hidden[:, None].expand(num_tokens, HIDDEN).contiguous()
mix = hidden[:, :HC].contiguous()
residual = hidden[:, None, :].expand(num_tokens, HC, HIDDEN).contiguous()
return (
hidden,
torch.arange(num_tokens, device=DEVICE),
None,
mix + 1,
mix + 2,
mix + 3,
residual,
)
def test_run_gathers_states_and_realigns_shared_indexer_buffers():
topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int()
candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int()
topk_before, candidates_before = topk.clone(), candidates.clone()
seen = {}
def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual):
seen["hidden_states"] = hidden_states.clone()
replay_context = get_forward_context()
seen["attn_metadata"] = replay_context.attn_metadata
seen["is_padding"] = replay_context.is_padding
return residual, pre_mix
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
states = _states(NUM_TOKENS)
hidden, residual = states[0], states[-1]
full, sub = object(), object()
_set_replay_batch(layers, REPLAY_ROWS, sub)
context = _context(full)
with override_forward_context(context):
outputs = layers(*states)
assert get_forward_context() is context
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
assert torch.equal(seen["hidden_states"], hidden[rows])
assert seen["attn_metadata"] is sub and seen["is_padding"] is None
assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows])
assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows])
assert len(outputs) == 2 and outputs[0].shape == residual.shape
assert torch.equal(outputs[0][rows], residual[rows])
assert outputs[0][1:173].abs().sum() == 0
def test_no_replay_batch_runs_the_whole_batch():
seen = {}
def run_layers(hidden_states, *rest):
seen["attn_metadata"] = get_forward_context().attn_metadata
return (hidden_states,)
layers = DecoderReplayLayers(WINDOW, run_layers, [])
states = _states(NUM_TOKENS)
full = object()
with override_forward_context(_context(full)):
(output,) = layers(*states)
assert output is states[0] and seen["attn_metadata"] is full
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The decoder-side SWA bounded replay batch DeepseekV41ModelState prepares."""
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="the replay buffers live on the GPU"
)
WINDOW = 128
DEVICE = torch.device("cuda")
GROUPS = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(
layer_names=["swa"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=False, prefix_replay_tokens=WINDOW
),
),
SimpleNamespace(
layer_names=["mla"],
kv_cache_spec=SimpleNamespace(
prefix_cacheable=True, prefix_replay_tokens=0
),
),
]
)
BLOCK_TABLES = (torch.zeros(4, 4, device=DEVICE),) * 2
# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300)
QUERY_LENS = [1, 300, 100]
SEQ_LENS = [500, 300, 300]
PREFILLING = [False, True, True]
REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)]
@pytest.fixture
def state(monkeypatch):
"""A DeepseekV41ModelState whose attention builds are recorded, not run."""
cfg = MagicMock()
cfg.model_config.enable_prompt_embeds = False
cfg.model_config.uses_mrope = False
cfg.model_config.is_multimodal_model = False
cfg.scheduler_config.max_num_seqs = 16
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
builds: list = []
def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw):
build = SimpleNamespace(
batch=input_batch,
cg_mode=cg_mode,
slot_mappings=slot_mappings.clone(),
replay_start=kw["model_specific_attn_metadata"].replay_start,
attn_metadata=object(),
)
builds.append(build)
return {"swa": build.attn_metadata}
monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn)
state = DeepseekV41ModelState(cfg, model, None, DEVICE)
state.builds = builds # type: ignore[attr-defined]
return state
def _input_batch(
query_lens: list[int],
seq_lens: list[int],
is_prefilling: list[bool],
device_query_lens: list[int] | None = None,
num_tokens_after_padding: int | None = None,
max_query_len: int | None = None,
) -> InputBatch:
num_reqs, num_tokens = len(query_lens), sum(query_lens)
num_padded = num_tokens_after_padding or num_tokens
batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(16, 1024, DEVICE))
return replace(
batch,
num_tokens=num_tokens,
num_tokens_after_padding=num_padded,
query_start_loc=torch.tensor(
[0, *np.cumsum(device_query_lens or query_lens)],
dtype=torch.int32,
device=DEVICE,
),
query_start_loc_np=np.array([0, *np.cumsum(query_lens)], dtype=np.int32),
seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE),
seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32),
is_prefilling_np=np.array(is_prefilling, dtype=np.bool_),
positions=torch.cat(
[torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)]
+ [torch.zeros(num_padded - num_tokens, dtype=torch.int64)]
).to(DEVICE),
is_padding=torch.arange(num_padded, device=DEVICE) >= num_tokens,
max_query_len=max_query_len,
)
def _slot_mappings(num_tokens: int) -> torch.Tensor:
return torch.stack([torch.arange(num_tokens), torch.arange(num_tokens) * 10]).to(
DEVICE
)
def _prepare(state, batch, cg_mode):
"""Runs prepare_attn; returns the replay batch and the replay build's inputs."""
state.prepare_attn(
batch,
cg_mode,
BLOCK_TABLES,
_slot_mappings(batch.num_tokens_after_padding),
[],
GROUPS,
)
layers = state.decoder_replay_layers
if layers.rows is None:
assert layers.forward_context is None
return None, None
return SimpleNamespace(
rows=layers.rows, forward_context=layers.forward_context
), state.builds[-1]
def test_replay_batch_keeps_each_request_window(state):
state._replay_start_np[2] = 50 # the encoder-side replay start of request 2
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
rows = torch.tensor(REPLAY_ROWS, device=DEVICE)
assert torch.equal(replay.rows, rows)
sub = build.batch
assert build.cg_mode == CUDAGraphMode.NONE
assert sub.query_start_loc.tolist() == [0, 1, 129, 229]
assert sub.query_start_loc_np.tolist() == [0, 1, 129, 229]
assert sub.num_tokens == sub.num_tokens_after_padding == 229
assert sub.max_query_len == WINDOW
# The trimmed request's window starts at its replay window; a higher
# encoder-side replay start stands.
assert build.replay_start.tolist() == [0, 300 - WINDOW, 50]
assert torch.equal(sub.positions, batch.positions[rows])
assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows])
context = replay.forward_context
assert context.attn_metadata == {"swa": build.attn_metadata}
assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows])
assert context.dp_metadata is None and context.is_padding is None
def test_replay_batch_keeps_device_decode_boundaries(state):
"""Adaptive verification resizes decodes on the GPU (CPU [2, 2], device
[1, 3]): their rows stay and the boundaries shift only by trimmed rows."""
batch = _input_batch(
[2, 2, 300], [10, 10, 300], PREFILLING, device_query_lens=[1, 3, 300]
)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == [0, 1, 2, 3, *range(304 - WINDOW, 304)]
assert build.batch.query_start_loc.tolist() == [0, 1, 4, 4 + WINDOW]
def test_prompt_logprobs_requests_keep_their_rows(state):
"""Every prompt row of a prompt-logprobs request is read, so it never
trims; the other prefills still do."""
state._keeps_rows_np[1] = True
batch = _input_batch([1, 300, 300], [500, 300, 300], PREFILLING)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == [0, *range(1, 301), *range(601 - WINDOW, 601)]
assert build.batch.max_query_len == 300
def test_replay_batch_keeps_adaptive_verification_query_bound(state):
"""Adaptive verification bounds the decodes' device-side query lengths
above their CPU lengths; the replay batch keeps that bound."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, max_query_len=200)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None and build.batch.max_query_len == 200
def test_only_eager_steps_trim(state):
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
padded batch to its real rows."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, batch, cg_mode) == (None, None)
replay, build = _prepare(state, batch, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == REPLAY_ROWS
assert build.cg_mode == CUDAGraphMode.NONE
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
def test_whole_batch_forwards_get_no_replay_batch(state):
short = _input_batch([100, 100], [100, 100], [True, True])
for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, short, cg_mode) == (None, None)
assert len(state.builds) == 3 # the batch's own metadata only
# Dummy batches, captures included (prepared as eager), are not prefills and
# keep their rows.
dummy = _input_batch(
QUERY_LENS, SEQ_LENS, [False] * 3, num_tokens_after_padding=512
)
assert _prepare(state, dummy, CUDAGraphMode.NONE) == (None, None)
@pytest.fixture
def dp_state(state, monkeypatch):
"""`state` on DP rank 0 of 2; `state.other` sets what rank 1 reports
(trims, replay tokens)."""
import vllm.models.deepseek_v41.nvidia.model_state as module
state.vllm_config.parallel_config.data_parallel_size = 2
state.vllm_config.parallel_config.data_parallel_rank = 0
state.vllm_config.parallel_config.is_moe_model = True
state.other = (False, 0) # type: ignore[attr-defined]
def all_reduce(tensor, group):
tensor[:, 1] = torch.tensor(state.other, dtype=torch.int32)
monkeypatch.setattr(module.dist, "all_reduce", all_reduce)
monkeypatch.setattr(module, "get_dp_group", lambda: SimpleNamespace(cpu_group=None))
return state
def test_dp_ranks_replay_together(dp_state):
"""A trimming rank makes the others replay too, on every rank's count."""
short = _input_batch([100, 100], [100, 100], [True, True])
dp_state.other = (False, 200)
assert _prepare(dp_state, short, CUDAGraphMode.NONE) == (None, None)
dp_state.other = (True, 300) # rank 1 trims to 300 rows
replay, build = _prepare(dp_state, short, CUDAGraphMode.NONE)
assert replay is not None
assert replay.rows.tolist() == list(range(200)) # this rank keeps every row
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [200, 300]
def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
dummy = _input_batch([512], [512], [False])
dp_state.other = (True, 129)
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
assert replay is not None and replay.rows.shape[0] == WINDOW
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
+2 -2
View File
@@ -260,7 +260,7 @@ def test_quantization(
model,
model_impl="auto",
enforce_eager=True,
**quantization_kwargs, # type: ignore[arg-type]
**quantization_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
@@ -270,7 +270,7 @@ def test_quantization(
model,
model_impl="transformers",
enforce_eager=True,
**quantization_kwargs, # type: ignore[arg-type]
**quantization_kwargs,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.using_transformers_backend()